fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation

- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.)
- Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics
- Inject surrounding paragraph context into translation API to avoid out-of-context translations
- Inject offline dictionary hints into LLM prompt for accurate contextual word meaning
- Prefer qwen2.5:3b for higher translation fluency and fidelity
- Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence
- Bump Service Worker cache to notes-v6.0
- Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
2026-10-06 22:32:36 +03:00
parent 3098d3016d
commit 532e2ec3b2
11 changed files with 313 additions and 89 deletions
+88
View File
@@ -1867,6 +1867,94 @@ def test_cefr_context_resolution_and_repair():
assert "negotiate" in repaired["context_sentence"].lower()
def test_robust_sentence_split_and_ai_translation(monkeypatch):
from app.miner import robust_sentence_split
from app.ai import build_translation_prompt
# 1. Test robust sentence segmentation
# Case A: Initials in names (John F. Kennedy)
s_jfk = "Perhaps the most famous is the John F. Kennedy International Airport in New York, but there is also the John F. Kennedy Space Center."
splits_jfk = robust_sentence_split(s_jfk)
assert len(splits_jfk) == 1
assert "John F. Kennedy International Airport" in splits_jfk[0]
# Case B: Honorifics with dots (Mr., Mrs., Dr., etc.)
s_dursley = "When Mr. and Mrs. Dursley woke up on the dull, gray Tuesday our story starts, there was nothing about the cloudy sky outside to suggest that strange things would happen."
splits_dursley = robust_sentence_split(s_dursley)
assert len(splits_dursley) == 1
assert "Mr. and Mrs. Dursley" in splits_dursley[0]
# Case C: Abbreviations (No. 4, U.S. Navy)
s_no4 = "They lived at No. 4 Privet Drive, which was a very quiet street. Next sentence starts here."
splits_no4 = robust_sentence_split(s_no4)
assert len(splits_no4) == 2
assert splits_no4[0] == "They lived at No. 4 Privet Drive, which was a very quiet street."
assert splits_no4[1] == "Next sentence starts here."
s_us = "The men thought that the U.S. Navy would send a plane to look for them. They waited patiently."
splits_us = robust_sentence_split(s_us)
assert len(splits_us) == 2
assert "U.S. Navy" in splits_us[0]
# Case D: Complex paragraph with Mr., Mrs., dialogue
s_hp = (
"At half past eight, Mr. Dursley picked up his briefcase, pecked Mrs. Dursley on the cheek, "
"and tried to kiss Dudley good-bye but missed, because Dudley was now having a tantrum and "
"throwing his cereal at the walls. “Little tyke,” chortled Mr. Dursley as he left the house."
)
splits_hp = robust_sentence_split(s_hp)
assert len(splits_hp) == 2
assert "pecked Mrs. Dursley on the cheek" in splits_hp[0]
assert splits_hp[0].startswith("At half past eight, Mr. Dursley")
assert "Little tyke" in splits_hp[1]
# 2. Test build_translation_prompt
prompt = build_translation_prompt(
sentence="At half past eight, Mr. Dursley picked up his briefcase.",
target_word="briefcase",
context="Full context paragraph here.",
dict_hint="портфель, кейс, дипломат",
)
assert "Контекст абзаца:" in prompt
assert "Full context paragraph here." in prompt
assert "briefcase" in prompt
assert "портфель, кейс, дипломат" in prompt
assert "Mr. -> мистер" in prompt
assert "НЕ дословным, без топорного калькирования" in prompt
# 3. Test API endpoint /api/translate-sentence
# Authenticate client
login_resp = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
assert login_resp.status_code == 200
# Mock translate_sentence_ai
import app.main as main_mod
async def mock_translate_sentence_ai(sentence, target_word="", context=""):
return {
"original_sentence": sentence,
"sentence_ru": "В половине девятого мистер Дурсли взял свой портфель.",
"target_word": target_word,
"word_meaning": "портфель",
"source": "mock_ollama",
"cached": False,
}
monkeypatch.setattr(main_mod, "translate_sentence_ai", mock_translate_sentence_ai)
tr_resp = client.post("/api/translate-sentence", json={
"sentence": "At half past eight, Mr. Dursley picked up his briefcase.",
"target_word": "briefcase",
"context": "Context paragraph",
})
assert tr_resp.status_code == 200
tr_data = tr_resp.json()
assert tr_data["sentence_ru"] == "В половине девятого мистер Дурсли взял свой портфель."
assert tr_data["word_meaning"] == "портфель"
assert tr_data["target_word"] == "briefcase"