fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation

- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.)
- Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics
- Inject surrounding paragraph context into translation API to avoid out-of-context translations
- Inject offline dictionary hints into LLM prompt for accurate contextual word meaning
- Prefer qwen2.5:3b for higher translation fluency and fidelity
- Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence
- Bump Service Worker cache to notes-v6.0
- Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
2026-10-06 22:32:36 +03:00
parent 3098d3016d
commit 532e2ec3b2
11 changed files with 313 additions and 89 deletions
+106 -37
View File
@@ -1,46 +1,81 @@
import json
import logging
import re
from typing import Optional, Dict, Any
import httpx
from app import config
from app.database import get_cached_sentence_translation, save_sentence_translation
from app.database import get_db, get_cached_sentence_translation, save_sentence_translation
logger = logging.getLogger(__name__)
def build_translation_prompt(sentence: str, target_word: str = "") -> str:
def get_dictionary_hint(term: str) -> str:
"""Retrieves verified offline dictionary translation hints for term."""
clean = term.strip()
if not clean:
return ""
try:
with get_db() as conn:
row = conn.execute(
"SELECT translation FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
(clean,),
).fetchone()
if row and row["translation"]:
return row["translation"].strip()
except Exception:
pass
return ""
def build_translation_prompt(
sentence: str,
target_word: str = "",
context: str = "",
dict_hint: str = "",
) -> str:
clean_sentence = sentence.strip()
clean_word = target_word.strip()
clean_context = context.strip()
context_section = ""
if clean_context and clean_context != clean_sentence:
short_ctx = clean_context[:500].strip()
context_section = f'Контекст абзаца:\n"{short_ctx}"\n\n'
target_section = ""
if clean_word:
return (
"You are a professional English-to-Russian translator.\n"
"Translate the following English sentence into clear, natural, and idiomatic Russian.\n"
f"Also provide the precise in-context meaning and translation of the target word/phrase '{clean_word}'.\n\n"
f'Sentence: "{clean_sentence}"\n'
f'Target word: "{clean_word}"\n\n'
"Respond ONLY with a valid JSON object in this format:\n"
"{\n"
' "sentence_ru": "перевод предложения на русский язык",\n'
' "word_meaning": "значение слова в этом контексте"\n'
"}"
)
else:
return (
"You are a professional English-to-Russian translator.\n"
"Translate the following English sentence into clear, natural, and idiomatic Russian.\n\n"
f'Sentence: "{clean_sentence}"\n\n'
"Respond ONLY with a valid JSON object in this format:\n"
"{\n"
' "sentence_ru": "перевод предложения на русский язык",\n'
' "word_meaning": ""\n'
"}"
)
target_section = f'Целевое слово/выражение: "{clean_word}"\n'
if dict_hint:
target_section += f'Возможные словарные значения: {dict_hint}\n'
target_section += "\n"
return (
"Ты — профессиональный литературный переводчик художественной и научно-популярной литературы на русский язык.\n"
"Твоя задача — перевести указанное английское предложение на красивый, естественный и грамотный русский литературный язык, идеально вписывающийся в повествование книги.\n\n"
f"{context_section}"
f'Предложение для перевода:\n"{clean_sentence}"\n\n'
f"{target_section}"
"Обязательные правила перевода:\n"
"1. Перевод должен быть живым и художественным (НЕ дословным, без топорного калькирования).\n"
"2. Переводи всё предложение ПОЛНОСТЬЮ от начала до конца, не обрывая на точках, обращениях или знаках препинания.\n"
"3. Обязательно сохраняй обращения и титулы: Mr. -> мистер, Mrs. -> миссис, Dr. -> доктор, Prof. -> профессор, St. -> святой/Сент-. Не теряй их!\n"
"4. В поле \"word_meaning\" укажи только точный перевод выделенного слова на русский язык в контексте данного предложения (выбрав самый подходящий вариант из словарных подсказок).\n\n"
"Ответь ИСКЛЮЧИТЕЛЬНО в формате JSON:\n"
"{\n"
' "sentence_ru": "полный литературный перевод всего предложения",\n'
f' "word_meaning": "{"русский перевод слова в данном контексте" if clean_word else ""}"\n'
"}"
)
async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[str, Any]:
async def translate_sentence_ai(
sentence: str,
target_word: str = "",
context: str = ""
) -> Dict[str, Any]:
"""
Translates a sentence using local AI (Ollama), with SQLite instant-cache lookup.
Incorporates paragraph context and offline dictionary hints for maximum naturalness.
Returns:
{
"original_sentence": "...",
@@ -53,6 +88,7 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st
"""
clean_sentence = sentence.strip()
clean_word = target_word.strip()
clean_context = context.strip()
if not clean_sentence:
return {
@@ -76,8 +112,17 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st
"cached": True,
}
# 2. Local AI Generation (Ollama)
prompt = build_translation_prompt(clean_sentence, clean_word)
# 2. Retrieve offline dictionary hint to steer LLM towards authentic meanings
dict_hint = get_dictionary_hint(clean_word) if clean_word else ""
# 3. Local AI Generation (Ollama)
prompt = build_translation_prompt(
sentence=clean_sentence,
target_word=clean_word,
context=clean_context,
dict_hint=dict_hint,
)
payload = {
"model": config.OLLAMA_MODEL,
"prompt": prompt,
@@ -85,7 +130,7 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st
"format": "json",
"options": {
"temperature": 0.2,
"num_predict": 256,
"num_predict": 300,
},
}
@@ -94,17 +139,15 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st
source = f"ollama:{config.OLLAMA_MODEL}"
try:
async with httpx.AsyncClient(timeout=30.0) as client:
async with httpx.AsyncClient(timeout=35.0) as client:
resp = await client.post(f"{config.OLLAMA_BASE_URL}/api/generate", json=payload)
if resp.status_code == 200:
data = resp.json()
raw_response = data.get("response", "{}")
try:
parsed = json.loads(raw_response)
sentence_ru = parsed.get("sentence_ru", "").strip()
word_meaning = parsed.get("word_meaning", "").strip()
except Exception:
# In case LLM emitted unescaped markdown or wrapper
# In case LLM emitted markdown wrapper ```json ... ```
clean_raw = raw_response.strip()
if clean_raw.startswith("```json"):
clean_raw = clean_raw[7:]
@@ -112,15 +155,41 @@ async def translate_sentence_ai(sentence: str, target_word: str = "") -> Dict[st
clean_raw = clean_raw[3:]
if clean_raw.endswith("```"):
clean_raw = clean_raw[:-3]
parsed = json.loads(clean_raw.strip())
sentence_ru = parsed.get("sentence_ru", "").strip()
word_meaning = parsed.get("word_meaning", "").strip()
try:
parsed = json.loads(clean_raw.strip())
except Exception:
parsed = {}
sentence_ru = str(parsed.get("sentence_ru", "")).strip()
# Extract word meaning flexibly
raw_meaning = (
parsed.get("word_meaning")
or parsed.get(f"{clean_word}_meaning")
or parsed.get("meaning")
or ""
)
raw_meaning = str(raw_meaning).strip()
# Clean up format like "briefcase: портфель" or "портфель (сущ.)"
if ":" in raw_meaning:
raw_meaning = raw_meaning.split(":", 1)[1].strip()
# If model returned the English term back, fallback to dictionary hint
if raw_meaning.lower() == clean_word.lower() or not raw_meaning:
if dict_hint:
raw_meaning = dict_hint.split(",")[0].strip()
else:
raw_meaning = ""
word_meaning = raw_meaning
else:
logger.warning(f"Ollama returned HTTP {resp.status_code}: {resp.text}")
except Exception as e:
logger.warning(f"Ollama connection error: {e}")
# 3. Graceful Fallback if Ollama is unreachable
# 4. Graceful Fallback if Ollama is unreachable
if not sentence_ru:
sentence_ru = "(Локальный ИИ временно недоступен)"
source = "fallback"
+10 -7
View File
@@ -2056,10 +2056,13 @@ def resolve_context_sentence_for_term(
if not clean_term:
return ""
# 1. If candidate_sentence already contains this term (case-insensitive word boundary)
# 1. If candidate_sentence already contains this term (and is a complete sentence)
if candidate_sentence:
cand_clean = candidate_sentence.strip()
if re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
is_complete = bool(re.search(r"[.!?]['\"”’»)]*$", cand_clean)) and not bool(
re.search(r"\b(?:Mr|Mrs|Ms|Miss|Dr|Prof|St|Jr|Sr|[A-Za-z])\.['\"”’»)]*$", cand_clean, re.IGNORECASE)
)
if is_complete and re.search(rf"\b{re.escape(clean_term)}\b", cand_clean, re.IGNORECASE):
return re.sub(r"\s+", " ", cand_clean.replace("___", clean_term)).strip()
# 2. Look up in cefr_questions specifically for this term
@@ -2075,7 +2078,7 @@ def resolve_context_sentence_for_term(
# 3. Look up in books table
try:
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 10").fetchall()
books = conn.execute("SELECT chapters_json FROM books ORDER BY id DESC LIMIT 50").fetchall()
for b in books:
try:
chapters = json.loads(b["chapters_json"])
@@ -2084,10 +2087,10 @@ def resolve_context_sentence_for_term(
for ch in chapters:
content = ch.get("content", "")
if clean_term.lower() in content.lower():
for s in re.split(r"(?<=[.!?])\s+", content):
s_clean = s.strip()
if 20 <= len(s_clean) <= 180 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
return re.sub(r"\s+", " ", s_clean).strip()
from app.miner import robust_sentence_split
for s_clean in robust_sentence_split(content):
if 20 <= len(s_clean) <= 240 and re.search(rf"\b{re.escape(clean_term)}\b", s_clean, re.IGNORECASE):
return s_clean.strip()
except Exception:
pass
+1 -1
View File
@@ -42,5 +42,5 @@ DB_PATH = Path(os.getenv("DB_PATH", str(BASE_DIR / "data" / "notes.db")))
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:3b")
+16 -1
View File
@@ -386,7 +386,8 @@ def _migrate(conn: sqlite3.Connection):
def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
"""
Repairs legacy vocabulary cards that were imported with placeholder blanks ('___')
or with mismatched contexts from CEFR test questions.
or with truncated/mismatched contexts from CEFR test questions and reader.
Also purges truncated sentence cache entries.
"""
try:
from app.cefr import resolve_context_sentence_for_term
@@ -396,11 +397,25 @@ def repair_mismatched_cefr_contexts(conn: sqlite3.Connection):
if not tbl:
return
# Purge truncated sentence cache entries
trans_tbl = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='sentence_translations'").fetchone()
if trans_tbl:
conn.execute(
"""
DELETE FROM sentence_translations
WHERE original_text LIKE '%pecked Mrs.%'
OR original_text LIKE '%the John F.'
OR instr(original_text, '___') > 0
"""
)
rows = conn.execute(
"""
SELECT id, term, context_sentence
FROM vocabulary_cards
WHERE instr(context_sentence, '___') > 0
OR context_sentence LIKE '%pecked Mrs.%'
OR context_sentence LIKE '%the John F.'
"""
).fetchall()
+2 -1
View File
@@ -190,6 +190,7 @@ class RfcFetchRequest(BaseModel):
class SentenceTranslateRequest(BaseModel):
sentence: str = Field(..., min_length=1)
target_word: Optional[str] = ""
context: Optional[str] = ""
class CefrAnswerRequest(BaseModel):
@@ -446,7 +447,7 @@ async def api_translate_sentence(
clean_s = req.sentence.strip()
if not clean_s:
raise HTTPException(status_code=400, detail="Sentence cannot be empty")
return await translate_sentence_ai(clean_s, req.target_word or "")
return await translate_sentence_ai(clean_s, req.target_word or "", req.context or "")
@app.get("/api/ai/status")
+47 -13
View File
@@ -102,31 +102,65 @@ for _lvl, _pos_map in CEFR_LEXICON.items():
_WORD_INDEX[_w_low] = []
_WORD_INDEX[_w_low].append((_lvl, _pos))
# Common abbreviations to prevent false splits
_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."]
def robust_sentence_split(text: str) -> List[str]:
"""
Splits English narrative prose into clean, complete sentences.
Protects decimals, acronyms, single-letter initials, honorifics, titles, and abbreviations.
"""
if not text:
return []
# Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
s = re.sub(r"(\d+)\.(\d+)", r"\1§DOT§\2", text)
# Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
s = re.sub(r"\b((?:[A-Za-z]\.){2,})", lambda m: m.group(1).replace(".", "§DOT§"), s)
# Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
s = re.sub(r'\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“\'‘«—])', r"\1§DOT§", s)
# Step 4: Protect common honorifics, titles, and abbreviations
abbr_pattern = (
r"\b("
r"Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|"
r"Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|"
r"etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|"
r"est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|"
r"fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|"
r"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
r")\."
)
s = re.sub(abbr_pattern, r"\1§DOT§", s, flags=re.IGNORECASE)
# Step 5: Protect ellipsis (... or ....)
s = re.sub(r"\.{2,}", lambda m: "§ELLIPSIS§" + ("§DOT§" * len(m.group(0))), s)
# Step 6: Protect any period followed by lowercase letter
s = re.sub(r"\.([\'\"”’»)]*\s+[a-zа-яё])", r"§DOT§\1", s)
# Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote
marked = re.sub(r'([.!?][\'\"”’»)]*)(\s+)(?=[A-Z0-9"“\'‘«—]|$)', r"\1§SEP§", s)
raw_sentences = [
re.sub(r"\s+", " ", part.replace("§DOT§", ".").replace("§ELLIPSIS§", "")).strip()
for part in marked.split("§SEP§")
]
return [sent for sent in raw_sentences if sent]
def split_into_clean_sentences(text: str) -> List[str]:
"""
Safely splits narrative prose into clean, grammatically complete sentences.
Protects honorifics and abbreviations from being split incorrectly.
Protects honorifics, initials, and abbreviations from being split incorrectly.
"""
if not text:
return []
# Protect common abbreviations and titles
protected = text
for abbr in _HONORIFICS:
protected = protected.replace(abbr, abbr.replace(".", "§DOT§"))
# Split on sentence-ending punctuation (.!?) followed by space or newline
raw_splits = re.split(r"(?<=[.!?])\s+", protected)
raw_splits = robust_sentence_split(text)
clean_sentences = []
for s in raw_splits:
restored = s.replace("§DOT§", ".").strip()
# Flatten multiple spaces and line breaks
flattened = re.sub(r"\s+", " ", restored)
flattened = re.sub(r"\s+", " ", s).strip()
# Basic validity filters:
# 1. Skip dialogue lines starting with quotation marks or dashes
+3 -2
View File
@@ -114,13 +114,14 @@ export const api = {
const resp = await fetch(`/api/detect-phrasal-verb?sentence=${encodeURIComponent(sentence)}&word=${encodeURIComponent(word)}`);
return resp.ok ? await resp.json() : { detected: false, phrasal_verb: null };
},
async translateSentence(sentence, targetWord = '') {
async translateSentence(sentence, targetWord = '', context = '') {
const resp = await fetch('/api/translate-sentence', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sentence: sentence,
target_word: targetWord
target_word: targetWord,
context: context
})
});
if (resp.ok) {
+38 -26
View File
@@ -1104,40 +1104,51 @@ export function renderInteractiveSpans(container, text, contextText, knownWordsM
});
}
const READER_HONORIFICS = [
'Mr.', 'Mrs.', 'Ms.', 'Dr.', 'Prof.', 'St.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'Jr.', 'Sr.',
'Capt.', 'Col.', 'Gen.', 'Lt.', 'Sgt.', 'Rev.', 'Hon.'
];
export function robustSentenceSplit(text) {
if (!text) return [];
let s = text;
// Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
s = s.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2');
// Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
s = s.replace(/\b((?:[A-Za-z]\.){2,})/g, (m) => m.replace(/\./g, '§DOT§'));
// Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
s = s.replace(/\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“'‘«—])/g, '$1§DOT§');
// Step 4: Protect common honorifics, titles, and abbreviations
const abbrPattern = /\b(Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec)\./gi;
s = s.replace(abbrPattern, '$1§DOT§');
// Step 5: Protect ellipsis (... or ....)
s = s.replace(/\.{2,}/g, (m) => '§ELLIPSIS§' + '§DOT§'.repeat(m.length));
// Step 6: Protect any period followed by lowercase letter (e.g. 'word. otherword')
s = s.replace(/\.([\'"”’»)]*\s+[a-zа-яё])/g, '§DOT§$1');
// Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote/start of dialog
const marked = s.replace(/([.!?]['"”’»)]*)(\s+)(?=[A-Z0-9"“'‘«—]|$)/g, '$1§SEP§');
return marked.split('§SEP§')
.map(part => part.replace(/§DOT§/g, '.').replace(/§ELLIPSIS§/g, '').trim())
.filter(Boolean);
}
export function extractContextSentence(fullText, word) {
if (!fullText) return '';
let protectedText = fullText;
for (const h of READER_HONORIFICS) {
if (protectedText.toLowerCase().includes(h.toLowerCase())) {
const safe = h.replace(/\./g, '§DOT§');
const reH = new RegExp(escapeRegex(h), 'gi');
protectedText = protectedText.replace(reH, safe);
}
}
// Protect decimal numbers (e.g. 3.14, 1.5)
protectedText = protectedText.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2');
// Split sentences on terminal punctuation (.!?) followed by optional quotes/brackets and whitespace
const marked = protectedText.replace(/([.!?]['"”’»)]*)\s+/g, '$1§SEP§');
const rawSentences = marked.split('§SEP§');
const sentences = robustSentenceSplit(fullText);
const cleanWord = (word || '').trim();
if (cleanWord) {
const escaped = escapeRegex(cleanWord);
const wordRe = new RegExp(`(?:^|\\W)${escaped}(?:$|\\W)`, 'i');
for (const s of rawSentences) {
for (const s of sentences) {
if (wordRe.test(s)) {
return s.replace(/§DOT§/g, '.').trim();
return s.replace(/\s+/g, ' ').trim();
}
}
}
return fullText.slice(0, 160).trim();
return sentences.length > 0 ? sentences[0].replace(/\s+/g, ' ').trim() : fullText.slice(0, 160).trim();
}
// -----------------------------------------------------------------------
@@ -1176,7 +1187,7 @@ export function positionWordPopup(targetElement) {
readerWordPopup.style.left = `${Math.round(left)}px`;
}
export async function openWordPopup(term, contextSentence, targetElement) {
export async function openWordPopup(term, contextSentence, targetElement, paragraphContext = '') {
const readerWordPopup = document.getElementById('reader-word-popup');
const readerSettingsPanel = document.getElementById('reader-settings-panel');
const popupTerm = document.getElementById('popup-term');
@@ -1202,6 +1213,7 @@ export async function openWordPopup(term, contextSentence, targetElement) {
popupCurrentData = {
term: cleanTerm,
contextSentence: contextSentence,
paragraphContext: paragraphContext,
targetElement: targetElement,
existingCard: existingCard,
};
@@ -1339,7 +1351,7 @@ export async function openWordPopup(term, contextSentence, targetElement) {
export async function handleWordClick(word, paragraphText, targetElement) {
const sentence = extractContextSentence(paragraphText, word);
openWordPopup(word, sentence, targetElement);
openWordPopup(word, sentence, targetElement, paragraphText);
}
function updatePopupPositionOnScroll() {
@@ -1965,7 +1977,7 @@ export function initReader() {
if (popupContextMeaningWrap) popupContextMeaningWrap.classList.add('hidden');
try {
const res = await api.lookup.translateSentence(ctxSentence, term);
const res = await api.lookup.translateSentence(ctxSentence, term, popupCurrentData.paragraphContext || '');
if (popupSentenceLoading) popupSentenceLoading.classList.add('hidden');
if (popupSentenceTranslationWrap) popupSentenceTranslationWrap.classList.remove('hidden');
+1 -1
View File
@@ -1,7 +1,7 @@
// Service Worker for Notes English PWA (Stage 2)
// Strategy: Cache-first for static assets, Network-first for API calls.
const CACHE_NAME = 'notes-v5.9';
const CACHE_NAME = 'notes-v6.0';
const OFFLINE_URL = '/';
const PRECACHE_URLS = [