diff --git a/app/ai.py b/app/ai.py index 9bc103a..808cddf 100644 --- a/app/ai.py +++ b/app/ai.py @@ -9,6 +9,36 @@ from app.database import get_db, get_cached_sentence_translation, save_sentence_ logger = logging.getLogger(__name__) +COMMON_STOPWORDS = { + "a", "an", "the", "in", "on", "at", "to", "for", "of", "with", "by", "from", + "up", "about", "into", "over", "after", "beneath", "under", "above", "is", "am", + "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", + "did", "and", "but", "if", "or", "because", "as", "until", "while", "this", "that", + "these", "those", "i", "you", "he", "she", "it", "we", "they", "them", "him", "her", + "us", "me", "my", "your", "his", "its", "our", "their", "not", "no", "nor", "can", + "could", "will", "would", "shall", "should", "may", "might", "must", "there", "here", + "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", "most", + "other", "some", "such", "than", "too", "very", "so", "s", "t", "d", "ll", "ve", "m" +} + + +def clean_dict_translation(raw: str) -> str: + """Extracts first 1-2 clean Cyrillic meanings, stripping phonetic / Latin noise.""" + if not raw: + return "" + clean = re.sub(r'/[^/]*$', '', raw) + parts = re.split(r'[,;]\s*', clean) + cyrillic_parts = [] + for p in parts: + p_clean = re.sub(r"[a-zA-Záéíóúýšžčñ']+", '', p).strip() + p_clean = re.sub(r'\s+', ' ', p_clean).strip(' /\\,;-()[]') + if p_clean and re.search(r'[\u0400-\u04FF]', p_clean) and p_clean not in cyrillic_parts: + cyrillic_parts.append(p_clean) + if len(cyrillic_parts) >= 2: + break + return ", ".join(cyrillic_parts) if cyrillic_parts else raw[:50].strip() + + def get_dictionary_hint(term: str) -> str: """Retrieves verified offline dictionary translation hints for term.""" clean = term.strip() @@ -21,17 +51,80 @@ def get_dictionary_hint(term: str) -> str: (clean,), ).fetchone() if row and row["translation"]: - return row["translation"].strip() + return clean_dict_translation(row["translation"]) except Exception: pass return "" +def get_sentence_dictionary_hints(sentence: str, target_word: str = "") -> str: + """Extracts verified offline dictionary translations for key words and phrases in sentence.""" + if not sentence: + return "" + words = re.findall(r"\b[a-zA-Z\'-]+\b", sentence) + if not words: + return "" + + candidates = set() + # Check 2-word contiguous phrases (collocations/idioms) + for i in range(len(words) - 1): + phrase = f"{words[i]} {words[i+1]}".lower() + candidates.add(phrase) + + # Check content words + for w in words: + wl = w.lower() + if len(wl) > 2 and wl not in COMMON_STOPWORDS: + candidates.add(wl) + + if not candidates: + return "" + + hints = [] + special_guidance = { + "owl": "owl — это сова/неясыть (ни в коем случае НЕ орёл!)", + "tawny owl": "tawny owl — это сова/неясыть (птица семейства совиных, ни в коем случае НЕ орёл!)", + } + + try: + with get_db() as conn: + placeholders = ",".join(["?"] * len(candidates)) + query = f"SELECT term, translation FROM offline_dictionary WHERE term IN ({placeholders}) COLLATE NOCASE" + rows = conn.execute(query, list(candidates)).fetchall() + + matched_dict = {} + for r in rows: + term_lower = r["term"].lower() + clean_trans = clean_dict_translation(r["translation"]) + if clean_trans: + matched_dict[term_lower] = clean_trans + + seen_terms = set() + for cand in sorted(matched_dict.keys(), key=lambda x: (0 if " " in x else 1, 0 if x == target_word.lower() else 1)): + if cand in seen_terms: + continue + seen_terms.add(cand) + trans = matched_dict[cand] + guidance = special_guidance.get(cand, "") + if guidance: + hints.append(f"- {cand}: {trans} (ВНИМАНИЕ: {guidance})") + else: + hints.append(f"- {cand}: {trans}") + + if len(hints) >= 8: + break + except Exception as e: + logger.warning(f"Error extracting sentence dictionary hints: {e}") + + return "\n".join(hints) + + def build_translation_prompt( sentence: str, target_word: str = "", context: str = "", dict_hint: str = "", + sentence_hints: str = "", ) -> str: clean_sentence = sentence.strip() clean_word = target_word.strip() @@ -49,18 +142,24 @@ def build_translation_prompt( target_section += f'Возможные словарные значения: {dict_hint}\n' target_section += "\n" + hints_section = "" + if sentence_hints: + hints_section = f"Словарные подсказки по ключевым словам предложения:\n{sentence_hints}\n\n" + return ( "Ты — профессиональный литературный переводчик художественной и научно-популярной литературы на русский язык.\n" "Твоя задача — перевести указанное английское предложение на красивый, естественный, богатый и грамматически безупречный русский язык, идеально вписывающийся в повествование книги.\n\n" f"{context_section}" f'Предложение для перевода:\n"{clean_sentence}"\n\n' f"{target_section}" + f"{hints_section}" "Обязательные правила перевода:\n" - "1. ПОЛНОТА И СВЯЗНОСТЬ: Переводи абсолютно ВСЕ смысловые части и придаточные обороты без пропусков! Обращай особое внимание на бессоюзные определительные обороты (zero relative clauses, например: 'the book I read' -> 'книга, которую я читал', 'Tuesday our story starts' -> 'вторник, с которого начинается наша история'). Ничего не выбрасывай!\n" - "2. БЕЗ ДОСЛОВНОГО КАЛЬКИРОВАНИЯ: Никаких топорных калек предлогов ('on Tuesday' -> 'во вторник', а не 'на вторнике'). Парные эпитеты ('dull, gray') переводи богатыми русскими синонимами ('пасмурный, серый'), не повторяя одно и то же слово.\n" - "3. ИМЕНА И ТИТУЛЫ: Имена персонажей и фамилии передавай полностью на русском языке (кириллицей), без латинских букв (Mr. and Mrs. Dursley -> мистер и миссис Дурсли). Сохраняй титулы: Mr. -> мистер, Mrs. -> миссис, Dr. -> доктор, Prof. -> профессор.\n" - "4. РУССКИЙ СИНТАКСИС: Сохраняй естественный русский порядок слов (обстоятельства времени и места ставь перед сказуемым: 'вскоре по всей стране начнут происходить...').\n" - f'5. В поле "word_meaning" укажи только точный перевод слова "{clean_word}" на русский язык в контексте этого предложения (выбрав лучший вариант из подсказок).\n\n' + "1. ПОЛНОТА И СВЯЗНОСТЬ: Переводи абсолютно ВСЕ смысловые части, обороты и определения без пропусков! Обращай особое внимание на цепочки прилагательных и определений (например: 'a large, tawny owl' -> 'крупная неясыть' или 'большая рыжевато-бурая сова' — ни в коем случае не пропускай 'large'!). Также переводи бессоюзные определительные обороты (zero relative clauses, например: 'Tuesday our story starts' -> 'вторник, с которого начинается наша история'). Ничего не выбрасывай!\n" + "2. БЕЗ ГАЛЛЮЦИНАЦИЙ И ТОЧНОСТЬ ЗНАЧЕНИЙ: Фауна, флора и объекты должны переводиться строго точно по смыслу и словарю (например: 'owl' — это сова/неясыть, ни в коем случае НЕ орёл!). Никаких смысловых подмен.\n" + "3. БЕЗ ДОСЛОВНОГО КАЛЬКИРОВАНИЯ: Никаких топорных калек предлогов ('on Tuesday' -> 'во вторник', а не 'на вторнике'). Парные эпитеты ('dull, gray') переводи богатыми русскими синонимами ('пасмурный, серый'), не повторяя одно и то же слово.\n" + "4. ИМЕНА И ТИТУЛЫ: Имена персонажей и фамилии передавай полностью на русском языке (кириллицей), без латинских букв (Mr. and Mrs. Dursley -> мистер и миссис Дурсли). Сохраняй титулы: Mr. -> мистер, Mrs. -> миссис, Dr. -> доктор, Prof. -> профессор.\n" + "5. РУССКИЙ СИНТАКСИС: Сохраняй естественный русский порядок слов (обстоятельства времени и места ставь перед сказуемым: 'вскоре по всей стране начнут происходить...').\n" + f'6. В поле "word_meaning" укажи только точный перевод слова "{clean_word}" на русский язык в контексте этого предложения (выбрав лучший вариант из подсказок).\n\n' "Ответь ИСКЛЮЧИТЕЛЬНО в формате JSON:\n" "{\n" ' "sentence_ru": "полный качественный литературный перевод всего предложения без пропусков",\n' @@ -113,8 +212,9 @@ async def translate_sentence_ai( "cached": True, } - # 2. Retrieve offline dictionary hint to steer LLM towards authentic meanings + # 2. Retrieve offline dictionary hints to steer LLM towards authentic meanings dict_hint = get_dictionary_hint(clean_word) if clean_word else "" + sentence_hints = get_sentence_dictionary_hints(clean_sentence, clean_word) # 3. Local AI Generation (Ollama) prompt = build_translation_prompt( @@ -122,6 +222,7 @@ async def translate_sentence_ai( target_word=clean_word, context=clean_context, dict_hint=dict_hint, + sentence_hints=sentence_hints, ) payload = { diff --git a/app/phrasal.py b/app/phrasal.py index af0cef8..3e7c343 100644 --- a/app/phrasal.py +++ b/app/phrasal.py @@ -46,7 +46,14 @@ IRREGULARS = { PRONOUNS_AND_OBJECTS = { "it", "him", "her", "them", "us", "me", "you", "this", "that", "these", "those", - "things", "something", "everything", "anything", "all", "back", "away" + "things", "something", "everything", "anything", "all" +} + +PHRASAL_PARTICLES = { + "about", "across", "ahead", "along", "apart", "around", "round", "aside", + "at", "away", "back", "by", "down", "for", "forth", "forward", "in", + "into", "off", "on", "onto", "out", "over", "past", "round", "through", + "to", "together", "under", "up", "upon", "with" } # Curated high-frequency English phrasal verbs (CEFR A2-B2) with clean translations @@ -326,20 +333,26 @@ def detect_phrasal_verb( } break - # Check offline dictionary - row = execute_query( - "SELECT term, translation FROM offline_dictionary WHERE term = ? COLLATE NOCASE", - (cand,), - ) - if row: - match_result = { - "term": row[0], - "raw": raw_match, - "translation": row[1], - "separable": False, - "matched_words": tokens[start : start + span_len], - } - break + # Check offline dictionary only for genuine phrasal verbs: + # 1. Candidate must end with a known particle (e.g. 'turn on', 'come up with') + # 2. pos in offline_dictionary must explicitly be a verb / verb phrase + cand_words = cand.split() + if len(cand_words) >= 2 and cand_words[-1] in PHRASAL_PARTICLES: + row = execute_query( + "SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", + (cand,), + ) + if row: + pos_val = (row[2] or "").lower() + if "verb" in pos_val: + match_result = { + "term": row[0], + "raw": raw_match, + "translation": row[1], + "separable": False, + "matched_words": tokens[start : start + span_len], + } + break if match_result: break if match_result: @@ -352,8 +365,10 @@ def detect_phrasal_verb( if lower_tokens[v_idx + 1] in PRONOUNS_AND_OBJECTS: p_idx = v_idx + 2 if idx in (v_idx, p_idx): - v_lemmas = lemmatize_word(lower_tokens[v_idx]) particle = lower_tokens[p_idx] + if particle not in PHRASAL_PARTICLES: + continue + v_lemmas = lemmatize_word(lower_tokens[v_idx]) raw_match = " ".join(tokens[v_idx : p_idx + 1]) for v in v_lemmas: cand = f"{v} {particle}" @@ -367,18 +382,20 @@ def detect_phrasal_verb( } break row = execute_query( - "SELECT term, translation FROM offline_dictionary WHERE term = ? COLLATE NOCASE", + "SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (cand,), ) if row: - match_result = { - "term": row[0], - "raw": raw_match, - "translation": row[1], - "separable": True, - "matched_words": [tokens[v_idx], tokens[p_idx]], - } - break + pos_val = (row[2] or "").lower() + if "verb" in pos_val: + match_result = { + "term": row[0], + "raw": raw_match, + "translation": row[1], + "separable": True, + "matched_words": [tokens[v_idx], tokens[p_idx]], + } + break if match_result: break if match_result: diff --git a/app/static/sw.js b/app/static/sw.js index cd52704..25b8f69 100644 --- a/app/static/sw.js +++ b/app/static/sw.js @@ -1,7 +1,7 @@ // Service Worker for Notes English PWA (Stage 2) // Strategy: Cache-first for static assets, Network-first for API calls. -const CACHE_NAME = 'notes-v6.0'; +const CACHE_NAME = 'notes-v6.1'; const OFFLINE_URL = '/'; const PRECACHE_URLS = [ diff --git a/tests/test_api.py b/tests/test_api.py index 7ae85ec..bc35398 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1955,6 +1955,79 @@ def test_robust_sentence_split_and_ai_translation(monkeypatch): assert tr_data["target_word"] == "briefcase" +def test_phrasal_verb_detection_no_false_positive_for_noun_phrase(): + from app.phrasal import detect_phrasal_verb + + # Seed temporary test dictionary with tawny owl + with get_db() as conn: + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("tawny owl", "неясыть, неясыть серая", "Noun"), + ) + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("owl", "сыч, сова, филин", "Noun"), + ) + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("tawny", "рыжевато-коричневый", "Adjective"), + ) + + sentence = "None of them noticed a large, tawny owl flutter past the window." + # tawny owl is a noun phrase (an animal species), NOT a phrasal verb! + assert detect_phrasal_verb(sentence, "owl") is None + assert detect_phrasal_verb(sentence, "tawny") is None + assert detect_phrasal_verb(sentence, "large") is None + + # Genuine phrasal verbs must still be detected + s2 = "None of them noticed him take off his coat." + pv2 = detect_phrasal_verb(s2, "take") + assert pv2 is not None + assert pv2["term"] == "take off" + + s3 = "He turned it off quietly." + pv3 = detect_phrasal_verb(s3, "turned") + assert pv3 is not None + assert pv3["term"] == "turn off" + + +def test_sentence_dictionary_hints_and_fauna_precision(): + from app.ai import get_sentence_dictionary_hints, build_translation_prompt + + with get_db() as conn: + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("tawny owl", "неясыть, неясыть серая", "Noun"), + ) + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("owl", "сыч, сова, филин", "Noun"), + ) + conn.execute( + "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', ?)", + ("large", "большой, крупный", "Adjective"), + ) + + sentence = "None of them noticed a large, tawny owl flutter past the window." + hints = get_sentence_dictionary_hints(sentence, "tawny") + assert "owl" in hints + assert "НЕ орёл" in hints + assert "large" in hints + + prompt = build_translation_prompt( + sentence=sentence, + target_word="tawny", + context="", + dict_hint="рыжевато-коричневый", + sentence_hints=hints, + ) + assert "Словарные подсказки по ключевым словам предложения" in prompt + assert "НЕ орёл" in prompt + assert "ПОЛНОТА И СВЯЗНОСТЬ" in prompt + assert "large" in prompt + + +