From 52db766b23d3bbccf7b0badc5488bad2e60fd68c Mon Sep 17 00:00:00 2001 From: Antigravity Date: Tue, 6 Oct 2026 23:08:54 +0300 Subject: [PATCH] fix(dictionary): strip Latin romanization debris from Russian translations and sanitize dictionary cache --- app/ai.py | 15 ++++-------- app/cefr.py | 7 +++--- app/database.py | 18 ++++++++++++++ app/dictionary.py | 61 ++++++++++++++++++++++++++++++++++++++++++++--- app/miner.py | 13 ++++------ app/static/sw.js | 2 +- tests/test_api.py | 20 ++++++++++++++++ 7 files changed, 109 insertions(+), 27 deletions(-) diff --git a/app/ai.py b/app/ai.py index 808cddf..400b3fa 100644 --- a/app/ai.py +++ b/app/ai.py @@ -26,17 +26,10 @@ def clean_dict_translation(raw: str) -> str: """Extracts first 1-2 clean Cyrillic meanings, stripping phonetic / Latin noise.""" if not raw: return "" - clean = re.sub(r'/[^/]*$', '', raw) - parts = re.split(r'[,;]\s*', clean) - cyrillic_parts = [] - for p in parts: - p_clean = re.sub(r"[a-zA-Záéíóúýšžčñ']+", '', p).strip() - p_clean = re.sub(r'\s+', ' ', p_clean).strip(' /\\,;-()[]') - if p_clean and re.search(r'[\u0400-\u04FF]', p_clean) and p_clean not in cyrillic_parts: - cyrillic_parts.append(p_clean) - if len(cyrillic_parts) >= 2: - break - return ", ".join(cyrillic_parts) if cyrillic_parts else raw[:50].strip() + from app.dictionary import clean_russian_translation + cleaned = clean_russian_translation(raw) + parts = [p.strip() for p in cleaned.split(",") if p.strip()] + return ", ".join(parts[:2]) if parts else raw[:50].strip() def get_dictionary_hint(term: str) -> str: diff --git a/app/cefr.py b/app/cefr.py index 1cd4855..c5ec349 100644 --- a/app/cefr.py +++ b/app/cefr.py @@ -15,6 +15,7 @@ import sqlite3 import random from typing import Dict, Any, List, Optional from datetime import datetime +from app.dictionary import clean_russian_translation CEFR_LEVELS = ["A1", "A2", "B1", "B2"] @@ -2190,7 +2191,7 @@ def process_cefr_answer( "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (selected_clean,), ).fetchone() - sel_trans = sel_row["translation"] if sel_row else selected_clean + sel_trans = clean_russian_translation(sel_row["translation"]) if sel_row else selected_clean sel_phon = sel_row["phonetic"] if sel_row else "" sel_pos = sel_row["pos"] if sel_row else "" sel_ctx = resolve_context_sentence_for_term(conn, selected_clean) @@ -2244,7 +2245,7 @@ def process_cefr_answer( "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (opt_clean,), ).fetchone() - opt_trans = opt_row["translation"] if opt_row else opt_clean + opt_trans = clean_russian_translation(opt_row["translation"]) if opt_row else opt_clean opt_phon = opt_row["phonetic"] if opt_row else "" opt_pos = opt_row["pos"] if opt_row else "" @@ -2413,7 +2414,7 @@ def ingest_cefr_word( "SELECT term, translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (clean_term,), ).fetchone() - trans = row["translation"] if row else clean_term + trans = clean_russian_translation(row["translation"]) if row else clean_term phon = row["phonetic"] if row else "" pos = row["pos"] if row else "" diff --git a/app/database.py b/app/database.py index 14e7661..b316f20 100644 --- a/app/database.py +++ b/app/database.py @@ -432,6 +432,22 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection): except Exception: pass + # Clean up any leftover Latin transliteration artifacts from offline_dictionary + try: + dirty_check = conn.execute( + "SELECT 1 FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*' LIMIT 1" + ).fetchone() + if dirty_check: + rows = conn.execute( + "SELECT rowid, translation FROM offline_dictionary WHERE translation GLOB '*[a-zA-Záéíóúýšžčñ/\\\\]*'" + ).fetchall() + from app.dictionary import clean_russian_translation + cleaned = [(clean_russian_translation(r[1]), r[0]) for r in rows] + conn.executemany("UPDATE offline_dictionary SET translation = ? WHERE rowid = ?", cleaned) + conn.execute("DELETE FROM dictionary_cache") + except Exception: + pass + def _seed_offline_dictionary(conn: sqlite3.Connection): """Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty.""" @@ -452,12 +468,14 @@ def _seed_offline_dictionary(conn: sqlite3.Connection): return import gzip + from app.dictionary import clean_russian_translation batch = [] with gzip.open(archive_path, "rt", encoding="utf-8") as f: for line in f: parts = line.rstrip("\r\n").split("\t") if len(parts) == 4: + parts[1] = clean_russian_translation(parts[1]) batch.append(parts) if len(batch) >= 10000: conn.executemany( diff --git a/app/dictionary.py b/app/dictionary.py index 6c1f621..cc7deb3 100644 --- a/app/dictionary.py +++ b/app/dictionary.py @@ -223,6 +223,60 @@ async def fetch_free_dictionary_api(word: str, client: Optional[httpx.AsyncClien pass return {"phonetic": None, "audio_url": None} +def clean_russian_translation(text: str) -> str: + """ + Cleans up Russian translations by stripping Latin romanization / phonetic debris + (e.g. 'át’sjaпорхать át’/', 'krýl’jami/', 'ževáto-koríčnevyj/', '] ]/]') + and restoring clean, deduplicated Cyrillic phrases, while preserving grammatical notes. + """ + if not text: + return "" + + # Protect grammatical notes like (прош. вр. от go), (3-е л. ед. ч. от flutter) + notes = [] + def save_note(m): + notes.append(m.group(0)) + return f"«ПРИМЕЧАНИЕ_{len(notes)-1}»" + + s = re.sub(r"\([а-яА-ЯёЁ0-9\s.,-]*\bот\s+[a-zA-Z\x27-]+\)", save_note, text) + + # 1. Clean out brackets junk and stray backslashes + s = re.sub(r"\[+[\s\]/\\,;]*\]*", " ", s) + s = re.sub(r"[\]\[\\]+", " ", s) + + # 2. Match Latin romanization fragments with accents/slashes/apostrophes + latin_rom = r"[a-zA-Z\u00C0-\u024F\u1E00-\u1EFF\x27\u2019/]+" + s = re.sub(rf"\s*{latin_rom}\s*", ", ", s) + + # 3. Clean up multiple commas/semicolons + s = re.sub(r"[,;\s]*,[,;\s]*", ", ", s) + + # 4. Clean and deduplicate phrases + items = [] + seen = set() + for part in s.split(","): + p = part.strip(" /\\,;-\x27\u2019") + if p.count("(") > p.count(")"): + p += ")" + elif p.count(")") > p.count("("): + p = re.sub(r"\)+$", "", p).strip() + p = re.sub(r"\s+", " ", p) + for idx, n in enumerate(notes): + token = f"«ПРИМЕЧАНИЕ_{idx}»" + if token in p: + p = p.replace(token, n).strip() + if p and re.search(r"[\u0400-\u04FF0-9]", p): + p_lower = p.lower() + if p_lower not in seen: + seen.add(p_lower) + items.append(p) + + res = ", ".join(items) if items else text.strip() + for idx, n in enumerate(notes): + res = res.replace(f"«ПРИМЕЧАНИЕ_{idx}»", n) + return res + + async def lookup_term(query: str) -> Dict[str, Any]: query_clean = query.strip() if not query_clean: @@ -251,13 +305,14 @@ async def lookup_term(query: str) -> Dict[str, Any]: ).fetchone() if off_row: raw_trans = off_row["translation"] or "" - tr_list = [t.strip() for t in raw_trans.split(",") if t.strip()] + clean_trans = clean_russian_translation(raw_trans) + tr_list = [t.strip() for t in clean_trans.split(",") if t.strip()] res = { "query": query_clean, "is_sentence": False, - "translation": raw_trans, + "translation": clean_trans, "translations_list": tr_list, - "translation_ru": raw_trans, + "translation_ru": clean_trans, "phonetic": off_row["phonetic"] or "", "audio_file": "", "audio_url": "", diff --git a/app/miner.py b/app/miner.py index cc1dffc..d8752dd 100644 --- a/app/miner.py +++ b/app/miner.py @@ -240,15 +240,10 @@ def clean_translation(trans: str) -> str: """Cleans up raw translation strings, removes slash phonetic residues and romanizations.""" if not trans: return "" - t = re.sub(r"/[^/]+/?", "", trans) - parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()] - clean_parts = [] - for p in parts: - if re.search(r"[а-яА-ЯёЁ]", p): - clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip() - if clean_p: - clean_parts.append(clean_p) - return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip() + from app.dictionary import clean_russian_translation + cleaned = clean_russian_translation(trans) + parts = [p.strip() for p in cleaned.split(",") if p.strip()] + return ", ".join(parts[:3]) if parts else trans.split(",")[0].strip() async def mine_book_questions( diff --git a/app/static/sw.js b/app/static/sw.js index 25b8f69..44fa93f 100644 --- a/app/static/sw.js +++ b/app/static/sw.js @@ -1,7 +1,7 @@ // Service Worker for Notes English PWA (Stage 2) // Strategy: Cache-first for static assets, Network-first for API calls. -const CACHE_NAME = 'notes-v6.1'; +const CACHE_NAME = 'notes-v6.2'; const OFFLINE_URL = '/'; const PRECACHE_URLS = [ diff --git a/tests/test_api.py b/tests/test_api.py index bc35398..5e1249a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2027,6 +2027,26 @@ def test_sentence_dictionary_hints_and_fauna_precision(): assert "large" in prompt +def test_clean_russian_translation_removes_romanization(): + from app.dictionary import clean_russian_translation + + raw_flutter = "трепет, порхание, развеваться át’sjaпорхать át’/, бить крыльями ’ krýl’jamiмахать крыльями át’ krýl’jami/" + clean_flutter = clean_russian_translation(raw_flutter) + assert clean_flutter == "трепет, порхание, развеваться, порхать, бить крыльями, махать крыльями" + assert "át’sja" not in clean_flutter + assert "krýl’jami" not in clean_flutter + assert "/" not in clean_flutter + + raw_tawny = "рыжевато-коричневый, рыжевато-коричневый ževáto-koríčnevyj/" + clean_tawny = clean_russian_translation(raw_tawny) + assert clean_tawny == "рыжевато-коричневый" + + raw_like = "подобный, похожий óžij/, как, типа ípa/" + clean_like = clean_russian_translation(raw_like) + assert clean_like == "подобный, похожий, как, типа" + + +