diff --git a/app/ai.py b/app/ai.py index bfbeb31..6c1f14e 100644 --- a/app/ai.py +++ b/app/ai.py @@ -1,46 +1,81 @@ import json import logging +import re from typing import Optional, Dict, Any import httpx from app import config -from app.database import get_cached_sentence_translation, save_sentence_translation +from app.database import get_db, get_cached_sentence_translation, save_sentence_translation logger = logging.getLogger(__name__) -def build_translation_prompt(sentence: str, target_word: str = "") -> str: +def get_dictionary_hint(term: str) -> str: + """Retrieves verified offline dictionary translation hints for term.""" + clean = term.strip() + if not clean: + return "" + try: + with get_db() as conn: + row = conn.execute( + "SELECT translation FROM offline_dictionary WHERE term = ? COLLATE NOCASE", + (clean,), + ).fetchone() + if row and row["translation"]: + return row["translation"].strip() + except Exception: + pass + return "" + + +def build_translation_prompt( + sentence: str, + target_word: str = "", + context: str = "", + dict_hint: str = "", +) -> str: clean_sentence = sentence.strip() clean_word = target_word.strip() + clean_context = context.strip() + context_section = "" + if clean_context and clean_context != clean_sentence: + short_ctx = clean_context[:500].strip() + context_section = f'Контекст абзаца:\n"{short_ctx}"\n\n' + + target_section = "" if clean_word: - return ( - "You are a professional English-to-Russian translator.\n" - "Translate the following English sentence into clear, natural, and idiomatic Russian.\n" - f"Also provide the precise in-context meaning and translation of the target word/phrase '{clean_word}'.\n\n" - f'Sentence: "{clean_sentence}"\n' - f'Target word: "{clean_word}"\n\n' - "Respond ONLY with a valid JSON object in this format:\n" - "{\n" - ' "sentence_ru": "перевод предложения на русский язык",\n' - ' "word_meaning": "значение слова в этом контексте"\n' - "}" - ) - else: - return ( - "You are a professional English-to-Russian translator.\n" - "Translate the following English sentence into clear, natural, and idiomatic Russian.\n\n" - f'Sentence: "{clean_sentence}"\n\n' - "Respond ONLY with a valid JSON object in this format:\n" - "{\n" - ' "sentence_ru": "перевод предложения на русский язык",\n' - ' "word_meaning": ""\n' - "}" - ) + target_section = f'Целевое слово/выражение: "{clean_word}"\n' + if dict_hint: + target_section += f'Возможные словарные значения: {dict_hint}\n' + target_section += "\n" + + return ( + "Ты — профессиональный литературный переводчик художественной и научно-популярной литературы на русский язык.\n" + "Твоя задача — перевести указанное английское предложение на красивый, естественный и грамотный русский литературный язык, идеально вписывающийся в повествование книги.\n\n" + f"{context_section}" + f'Предложение для перевода:\n"{clean_sentence}"\n\n' + f"{target_section}" + "Обязательные правила перевода:\n" + "1. Перевод должен быть живым и художественным (НЕ дословным, без топорного калькирования).\n" + "2. Переводи всё предложение ПОЛНОСТЬЮ от начала до конца, не обрывая на точках, обращениях или знаках препинания.\n" + "3. Обязательно сохраняй обращения и титулы: Mr. -> мистер, Mrs. -> миссис, Dr. -> доктор, Prof. -> профессор, St. -> святой/Сент-. Не теряй их!\n" + "4. В поле \"word_meaning\" укажи только точный перевод выделенного слова на русский язык в контексте данного предложения (выбрав самый подходящий вариант из словарных подсказок).\n\n" + "Ответь ИСКЛЮЧИТЕЛЬНО в формате JSON:\n" + "{\n" + ' "sentence_ru": "полный литературный перевод всего предложения",\n' + f' "word_meaning": "{"русский перевод слова в данном контексте" if clean_word else ""}"\n' + "}" + ) -async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[str, Any]: +async def translate_sentence_ai( + sentence: str, + target_word: str = "", + context: str = "" +) -> Dict[str, Any]: """ Translates a sentence using local AI (Ollama), with SQLite instant-cache lookup. + Incorporates paragraph context and offline dictionary hints for maximum naturalness. Returns: { "original_sentence": "...", @@ -53,6 +88,7 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st """ clean_sentence = sentence.strip() clean_word = target_word.strip() + clean_context = context.strip() if not clean_sentence: return { @@ -76,8 +112,17 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st "cached": True, } - # 2. Local AI Generation (Ollama) - prompt = build_translation_prompt(clean_sentence, clean_word) + # 2. Retrieve offline dictionary hint to steer LLM towards authentic meanings + dict_hint = get_dictionary_hint(clean_word) if clean_word else "" + + # 3. Local AI Generation (Ollama) + prompt = build_translation_prompt( + sentence=clean_sentence, + target_word=clean_word, + context=clean_context, + dict_hint=dict_hint, + ) + payload = { "model": config.OLLAMA_MODEL, "prompt": prompt, @@ -85,7 +130,7 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st "format": "json", "options": { "temperature": 0.2, - "num_predict": 256, + "num_predict": 300, }, } @@ -94,17 +139,15 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st source = f"ollama:{config.OLLAMA_MODEL}" try: - async with httpx.AsyncClient(timeout=30.0) as client: + async with httpx.AsyncClient(timeout=35.0) as client: resp = await client.post(f"{config.OLLAMA_BASE_URL}/api/generate", json=payload) if resp.status_code == 200: data = resp.json() raw_response = data.get("response", "{}") try: parsed = json.loads(raw_response) - sentence_ru = parsed.get("sentence_ru", "").strip() - word_meaning = parsed.get("word_meaning", "").strip() except Exception: - # In case LLM emitted unescaped markdown or wrapper + # In case LLM emitted markdown wrapper ```json ... ``` clean_raw = raw_response.strip() if clean_raw.startswith("```json"): clean_raw = clean_raw[7:] @@ -112,15 +155,41 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st clean_raw = clean_raw[3:] if clean_raw.endswith("```"): clean_raw = clean_raw[:-3] - parsed = json.loads(clean_raw.strip()) - sentence_ru = parsed.get("sentence_ru", "").strip() - word_meaning = parsed.get("word_meaning", "").strip() + try: + parsed = json.loads(clean_raw.strip()) + except Exception: + parsed = {} + + sentence_ru = str(parsed.get("sentence_ru", "")).strip() + + # Extract word meaning flexibly + raw_meaning = ( + parsed.get("word_meaning") + or parsed.get(f"{clean_word}_meaning") + or parsed.get("meaning") + or "" + ) + raw_meaning = str(raw_meaning).strip() + + # Clean up format like "briefcase: портфель" or "портфель (сущ.)" + if ":" in raw_meaning: + raw_meaning = raw_meaning.split(":", 1)[1].strip() + + # If model returned the English term back, fallback to dictionary hint + if raw_meaning.lower() == clean_word.lower() or not raw_meaning: + if dict_hint: + raw_meaning = dict_hint.split(",")[0].strip() + else: + raw_meaning = "" + + word_meaning = raw_meaning + else: logger.warning(f"Ollama returned HTTP {resp.status_code}: {resp.text}") except Exception as e: logger.warning(f"Ollama connection error: {e}") - # 3. Graceful Fallback if Ollama is unreachable + # 4. Graceful Fallback if Ollama is unreachable if not sentence_ru: sentence_ru = "(Локальный ИИ временно недоступен)" source = "fallback" diff --git a/app/cefr.py b/app/cefr.py index e6f0c9a..1cd4855 100644 --- a/app/cefr.py +++ b/app/cefr.py @@ -2056,10 +2056,13 @@ def resolve_context_sentence_for_term( if not clean_term: return "" - # 1. If candidate_sentence already contains this term (case-insensitive word boundary) + # 1. If candidate_sentence already contains this term (and is a complete sentence) if candidate_sentence: cand_clean = candidate_sentence.strip() - if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE): + is_complete = bool(re.search(r"[.!?]['\"”’»)]*$", cand_clean)) and not bool( + re.search(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|St|Jr|Sr|[A-Za-z])\.['\"”’»)]*$", cand_clean, re.IGNORECASE) + ) + if is_complete and re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE): return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip() # 2. Look up in cefr_questions specifically for this term @@ -2075,7 +2078,7 @@ def resolve_context_sentence_for_term( # 3. Look up in books table try: - books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall() + books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 50").fetchall() for b in books: try: chapters = json.loads(b["chapters_json"]) @@ -2084,10 +2087,10 @@ def resolve_context_sentence_for_term( for ch in chapters: content = ch.get("content", "") if clean_term.lower() in content.lower(): - for s in re.split(r"(?<=[.!?])\s+", content): - s_clean = s.strip() - if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE): - return re.sub(r"\s+", " ", s_clean).strip() + from app.miner import robust_sentence_split + for s_clean in robust_sentence_split(content): + if 20 <= len(s_clean) <= 240 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE): + return s_clean.strip() except Exception: pass diff --git a/app/config.py b/app/config.py index 79caca4..f588f0f 100644 --- a/app/config.py +++ b/app/config.py @@ -42,5 +42,5 @@ DB_PATH = Path(os.getenv("DB_PATH", str(BASE_DIR / "data" / "notes.db"))) DB_PATH.parent.mkdir(parents=True, exist_ok=True) OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/") -OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b") +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:3b") diff --git a/app/database.py b/app/database.py index ee3d094..14e7661 100644 --- a/app/database.py +++ b/app/database.py @@ -386,7 +386,8 @@ def _migrate(conn: sqlite3.Connection): def repair_mismatched_cefr_contexts(conn: sqlite3.Connection): """ Repairs legacy vocabulary cards that were imported with placeholder blanks ('___') - or with mismatched contexts from CEFR test questions. + or with truncated/mismatched contexts from CEFR test questions and reader. + Also purges truncated sentence cache entries. """ try: from app.cefr import resolve_context_sentence_for_term @@ -396,11 +397,25 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection): if not tbl: return + # Purge truncated sentence cache entries + trans_tbl = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='sentence_translations'").fetchone() + if trans_tbl: + conn.execute( + """ + DELETE FROM sentence_translations + WHERE original_text LIKE '%pecked Mrs.%' + OR original_text LIKE '%the John F.' + OR instr(original_text, '___') > 0 + """ + ) + rows = conn.execute( """ SELECT id, term, context_sentence FROM vocabulary_cards WHERE instr(context_sentence, '___') > 0 + OR context_sentence LIKE '%pecked Mrs.%' + OR context_sentence LIKE '%the John F.' """ ).fetchall() diff --git a/app/main.py b/app/main.py index 72075a9..ecbbbd9 100644 --- a/app/main.py +++ b/app/main.py @@ -190,6 +190,7 @@ class RfcFetchRequest(BaseModel): class SentenceTranslateRequest(BaseModel): sentence: str = Field(..., min_length=1) target_word: Optional[str] = "" + context: Optional[str] = "" class CefrAnswerRequest(BaseModel): @@ -446,7 +447,7 @@ async def api_translate_sentence( clean_s = req.sentence.strip() if not clean_s: raise HTTPException(status_code=400, detail="Sentence cannot be empty") - return await translate_sentence_ai(clean_s, req.target_word or "") + return await translate_sentence_ai(clean_s, req.target_word or "", req.context or "") @app.get("/api/ai/status") diff --git a/app/miner.py b/app/miner.py index 39d04a4..cc1dffc 100644 --- a/app/miner.py +++ b/app/miner.py @@ -102,31 +102,65 @@ for _lvl, _pos_map in CEFR_LEXICON.items(): _WORD_INDEX[_w_low] = [] _WORD_INDEX[_w_low].append((_lvl, _pos)) -# Common abbreviations to prevent false splits -_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."] +def robust_sentence_split(text: str) -> List[str]: + """ + Splits English narrative prose into clean, complete sentences. + Protects decimals, acronyms, single-letter initials, honorifics, titles, and abbreviations. + """ + if not text: + return [] + + # Step 1: Protect decimal numbers (e.g. 3.14, 1.5) + s = re.sub(r"(\d+)\.(\d+)", r"\1§DOT§\2", text) + + # Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D. + s = re.sub(r"\b((?:[A-Za-z]\.){2,})", lambda m: m.group(1).replace(".", "§DOT§"), s) + + # Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling' + s = re.sub(r'\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“\'‘«—])', r"\1§DOT§", s) + + # Step 4: Protect common honorifics, titles, and abbreviations + abbr_pattern = ( + r"\b(" + r"Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|" + r"Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|" + r"etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|" + r"est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|" + r"fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|" + r"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec" + r")\." + ) + s = re.sub(abbr_pattern, r"\1§DOT§", s, flags=re.IGNORECASE) + + # Step 5: Protect ellipsis (... or ....) + s = re.sub(r"\.{2,}", lambda m: "§ELLIPSIS§" + ("§DOT§" * len(m.group(0))), s) + + # Step 6: Protect any period followed by lowercase letter + s = re.sub(r"\.([\'\"”’»)]*\s+[a-zа-яё])", r"§DOT§\1", s) + + # Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote + marked = re.sub(r'([.!?][\'\"”’»)]*)(\s+)(?=[A-Z0-9"“\'‘«—]|$)', r"\1§SEP§", s) + + raw_sentences = [ + re.sub(r"\s+", " ", part.replace("§DOT§", ".").replace("§ELLIPSIS§", "")).strip() + for part in marked.split("§SEP§") + ] + return [sent for sent in raw_sentences if sent] def split_into_clean_sentences(text: str) -> List[str]: """ Safely splits narrative prose into clean, grammatically complete sentences. - Protects honorifics and abbreviations from being split incorrectly. + Protects honorifics, initials, and abbreviations from being split incorrectly. """ if not text: return [] - # Protect common abbreviations and titles - protected = text - for abbr in _HONORIFICS: - protected = protected.replace(abbr, abbr.replace(".", "§DOT§")) - - # Split on sentence-ending punctuation (.!?) followed by space or newline - raw_splits = re.split(r"(?<=[.!?])\s+", protected) + raw_splits = robust_sentence_split(text) clean_sentences = [] for s in raw_splits: - restored = s.replace("§DOT§", ".").strip() - # Flatten multiple spaces and line breaks - flattened = re.sub(r"\s+", " ", restored) + flattened = re.sub(r"\s+", " ", s).strip() # Basic validity filters: # 1. Skip dialogue lines starting with quotation marks or dashes diff --git a/app/static/js/modules/api.js b/app/static/js/modules/api.js index 19fc0aa..63a3c70 100644 --- a/app/static/js/modules/api.js +++ b/app/static/js/modules/api.js @@ -114,13 +114,14 @@ export const api = { const resp = await fetch(`/api/detect-phrasal-verb?sentence=${encodeURIComponent(sentence)}&word=${encodeURIComponent(word)}`); return resp.ok ? await resp.json() : { detected: false, phrasal_verb: null }; }, - async translateSentence(sentence, targetWord = '') { + async translateSentence(sentence, targetWord = '', context = '') { const resp = await fetch('/api/translate-sentence', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sentence: sentence, - target_word: targetWord + target_word: targetWord, + context: context }) }); if (resp.ok) { diff --git a/app/static/js/modules/reader.js b/app/static/js/modules/reader.js index 153933b..8852032 100644 --- a/app/static/js/modules/reader.js +++ b/app/static/js/modules/reader.js @@ -1104,40 +1104,51 @@ export function renderInteractiveSpans(container, text, contextText, knownWordsM }); } -const READER_HONORIFICS = [ - 'Mr.', 'Mrs.', 'Ms.', 'Dr.', 'Prof.', 'St.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'Jr.', 'Sr.', - 'Capt.', 'Col.', 'Gen.', 'Lt.', 'Sgt.', 'Rev.', 'Hon.' -]; +export function robustSentenceSplit(text) { + if (!text) return []; + let s = text; + + // Step 1: Protect decimal numbers (e.g. 3.14, 1.5) + s = s.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2'); + + // Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D. + s = s.replace(/\b((?:[A-Za-z]\.){2,})/g, (m) => m.replace(/\./g, '§DOT§')); + + // Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling' + s = s.replace(/\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“'‘«—])/g, '$1§DOT§'); + + // Step 4: Protect common honorifics, titles, and abbreviations + const abbrPattern = /\b(Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec)\./gi; + s = s.replace(abbrPattern, '$1§DOT§'); + + // Step 5: Protect ellipsis (... or ....) + s = s.replace(/\.{2,}/g, (m) => '§ELLIPSIS§' + '§DOT§'.repeat(m.length)); + + // Step 6: Protect any period followed by lowercase letter (e.g. 'word. otherword') + s = s.replace(/\.([\'"”’»)]*\s+[a-zа-яё])/g, '§DOT§$1'); + + // Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote/start of dialog + const marked = s.replace(/([.!?]['"”’»)]*)(\s+)(?=[A-Z0-9"“'‘«—]|$)/g, '$1§SEP§'); + + return marked.split('§SEP§') + .map(part => part.replace(/§DOT§/g, '.').replace(/§ELLIPSIS§/g, '').trim()) + .filter(Boolean); +} export function extractContextSentence(fullText, word) { if (!fullText) return ''; - let protectedText = fullText; - for (const h of READER_HONORIFICS) { - if (protectedText.toLowerCase().includes(h.toLowerCase())) { - const safe = h.replace(/\./g, '§DOT§'); - const reH = new RegExp(escapeRegex(h), 'gi'); - protectedText = protectedText.replace(reH, safe); - } - } - // Protect decimal numbers (e.g. 3.14, 1.5) - protectedText = protectedText.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2'); - - // Split sentences on terminal punctuation (.!?) followed by optional quotes/brackets and whitespace - const marked = protectedText.replace(/([.!?]['"”’»)]*)\s+/g, '$1§SEP§'); - const rawSentences = marked.split('§SEP§'); - + const sentences = robustSentenceSplit(fullText); const cleanWord = (word || '').trim(); if (cleanWord) { const escaped = escapeRegex(cleanWord); const wordRe = new RegExp(`(?:^|\\W)${escaped}(?:$|\\W)`, 'i'); - for (const s of rawSentences) { + for (const s of sentences) { if (wordRe.test(s)) { - return s.replace(/§DOT§/g, '.').trim(); + return s.replace(/\s+/g, ' ').trim(); } } } - - return fullText.slice(0, 160).trim(); + return sentences.length > 0 ? sentences[0].replace(/\s+/g, ' ').trim() : fullText.slice(0, 160).trim(); } // ----------------------------------------------------------------------- @@ -1176,7 +1187,7 @@ export function positionWordPopup(targetElement) { readerWordPopup.style.left = `${Math.round(left)}px`; } -export async function openWordPopup(term, contextSentence, targetElement) { +export async function openWordPopup(term, contextSentence, targetElement, paragraphContext = '') { const readerWordPopup = document.getElementById('reader-word-popup'); const readerSettingsPanel = document.getElementById('reader-settings-panel'); const popupTerm = document.getElementById('popup-term'); @@ -1202,6 +1213,7 @@ export async function openWordPopup(term, contextSentence, targetElement) { popupCurrentData = { term: cleanTerm, contextSentence: contextSentence, + paragraphContext: paragraphContext, targetElement: targetElement, existingCard: existingCard, }; @@ -1339,7 +1351,7 @@ export async function openWordPopup(term, contextSentence, targetElement) { export async function handleWordClick(word, paragraphText, targetElement) { const sentence = extractContextSentence(paragraphText, word); - openWordPopup(word, sentence, targetElement); + openWordPopup(word, sentence, targetElement, paragraphText); } function updatePopupPositionOnScroll() { @@ -1965,7 +1977,7 @@ export function initReader() { if (popupContextMeaningWrap) popupContextMeaningWrap.classList.add('hidden'); try { - const res = await api.lookup.translateSentence(ctxSentence, term); + const res = await api.lookup.translateSentence(ctxSentence, term, popupCurrentData.paragraphContext || ''); if (popupSentenceLoading) popupSentenceLoading.classList.add('hidden'); if (popupSentenceTranslationWrap) popupSentenceTranslationWrap.classList.remove('hidden'); diff --git a/app/static/sw.js b/app/static/sw.js index da07bde..cd52704 100644 --- a/app/static/sw.js +++ b/app/static/sw.js @@ -1,7 +1,7 @@ // Service Worker for Notes English PWA (Stage 2) // Strategy: Cache-first for static assets, Network-first for API calls. -const CACHE_NAME = 'notes-v5.9'; +const CACHE_NAME = 'notes-v6.0'; const OFFLINE_URL = '/'; const PRECACHE_URLS = [ diff --git a/docker-compose.yml b/docker-compose.yml index a95b8dc..19c80a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - ADMIN_USERNAME=kopri - DB_PATH=/app/data/notes.db - OLLAMA_BASE_URL=http://ollama:11434 + - OLLAMA_MODEL=qwen2.5:3b depends_on: - ollama diff --git a/tests/test_api.py b/tests/test_api.py index 38ef067..1451863 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1867,6 +1867,94 @@ def test_cefr_context_resolution_and_repair(): assert "negotiate" in repaired["context_sentence"].lower() +def test_robust_sentence_split_and_ai_translation(monkeypatch): + from app.miner import robust_sentence_split + from app.ai import build_translation_prompt + + # 1. Test robust sentence segmentation + # Case A: Initials in names (John F. Kennedy) + s_jfk = "Perhaps the most famous is the John F. Kennedy International Airport in New York, but there is also the John F. Kennedy Space Center." + splits_jfk = robust_sentence_split(s_jfk) + assert len(splits_jfk) == 1 + assert "John F. Kennedy International Airport" in splits_jfk[0] + + # Case B: Honorifics with dots (Mr., Mrs., Dr., etc.) + s_dursley = "When Mr. and Mrs. Dursley woke up on the dull, gray Tuesday our story starts, there was nothing about the cloudy sky outside to suggest that strange things would happen." + splits_dursley = robust_sentence_split(s_dursley) + assert len(splits_dursley) == 1 + assert "Mr. and Mrs. Dursley" in splits_dursley[0] + + # Case C: Abbreviations (No. 4, U.S. Navy) + s_no4 = "They lived at No. 4 Privet Drive, which was a very quiet street. Next sentence starts here." + splits_no4 = robust_sentence_split(s_no4) + assert len(splits_no4) == 2 + assert splits_no4[0] == "They lived at No. 4 Privet Drive, which was a very quiet street." + assert splits_no4[1] == "Next sentence starts here." + + s_us = "The men thought that the U.S. Navy would send a plane to look for them. They waited patiently." + splits_us = robust_sentence_split(s_us) + assert len(splits_us) == 2 + assert "U.S. Navy" in splits_us[0] + + # Case D: Complex paragraph with Mr., Mrs., dialogue + s_hp = ( + "At half past eight, Mr. Dursley picked up his briefcase, pecked Mrs. Dursley on the cheek, " + "and tried to kiss Dudley good-bye but missed, because Dudley was now having a tantrum and " + "throwing his cereal at the walls. “Little tyke,” chortled Mr. Dursley as he left the house." + ) + splits_hp = robust_sentence_split(s_hp) + assert len(splits_hp) == 2 + assert "pecked Mrs. Dursley on the cheek" in splits_hp[0] + assert splits_hp[0].startswith("At half past eight, Mr. Dursley") + assert "Little tyke" in splits_hp[1] + + # 2. Test build_translation_prompt + prompt = build_translation_prompt( + sentence="At half past eight, Mr. Dursley picked up his briefcase.", + target_word="briefcase", + context="Full context paragraph here.", + dict_hint="портфель, кейс, дипломат", + ) + assert "Контекст абзаца:" in prompt + assert "Full context paragraph here." in prompt + assert "briefcase" in prompt + assert "портфель, кейс, дипломат" in prompt + assert "Mr. -> мистер" in prompt + assert "НЕ дословным, без топорного калькирования" in prompt + + # 3. Test API endpoint /api/translate-sentence + # Authenticate client + login_resp = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"}) + assert login_resp.status_code == 200 + + # Mock translate_sentence_ai + import app.main as main_mod + + async def mock_translate_sentence_ai(sentence, target_word="", context=""): + return { + "original_sentence": sentence, + "sentence_ru": "В половине девятого мистер Дурсли взял свой портфель.", + "target_word": target_word, + "word_meaning": "портфель", + "source": "mock_ollama", + "cached": False, + } + + monkeypatch.setattr(main_mod, "translate_sentence_ai", mock_translate_sentence_ai) + + tr_resp = client.post("/api/translate-sentence", json={ + "sentence": "At half past eight, Mr. Dursley picked up his briefcase.", + "target_word": "briefcase", + "context": "Context paragraph", + }) + assert tr_resp.status_code == 200 + tr_data = tr_resp.json() + assert tr_data["sentence_ru"] == "В половине девятого мистер Дурсли взял свой портфель." + assert tr_data["word_meaning"] == "портфель" + assert tr_data["target_word"] == "briefcase" + + +