fix(ai,reader): robust sentence segmentation without abbreviation cutoffs & context-aware literary translation
- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.) - Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics - Inject surrounding paragraph context into translation API to avoid out-of-context translations - Inject offline dictionary hints into LLM prompt for accurate contextual word meaning - Prefer qwen2.5:3b for higher translation fluency and fidelity - Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence - Bump Service Worker cache to notes-v6.0 - Add unit tests for robust sentence splitting, prompt formatting, and translation API
This commit is contained in:
@@ -114,13 +114,14 @@ export const api = {
|
||||
const resp = await fetch(`/api/detect-phrasal-verb?sentence=${encodeURIComponent(sentence)}&word=${encodeURIComponent(word)}`);
|
||||
return resp.ok ? await resp.json() : { detected: false, phrasal_verb: null };
|
||||
},
|
||||
async translateSentence(sentence, targetWord = '') {
|
||||
async translateSentence(sentence, targetWord = '', context = '') {
|
||||
const resp = await fetch('/api/translate-sentence', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
sentence: sentence,
|
||||
target_word: targetWord
|
||||
target_word: targetWord,
|
||||
context: context
|
||||
})
|
||||
});
|
||||
if (resp.ok) {
|
||||
|
||||
@@ -1104,40 +1104,51 @@ export function renderInteractiveSpans(container, text, contextText, knownWordsM
|
||||
});
|
||||
}
|
||||
|
||||
const READER_HONORIFICS = [
|
||||
'Mr.', 'Mrs.', 'Ms.', 'Dr.', 'Prof.', 'St.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'Jr.', 'Sr.',
|
||||
'Capt.', 'Col.', 'Gen.', 'Lt.', 'Sgt.', 'Rev.', 'Hon.'
|
||||
];
|
||||
export function robustSentenceSplit(text) {
|
||||
if (!text) return [];
|
||||
let s = text;
|
||||
|
||||
// Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
|
||||
s = s.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2');
|
||||
|
||||
// Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
|
||||
s = s.replace(/\b((?:[A-Za-z]\.){2,})/g, (m) => m.replace(/\./g, '§DOT§'));
|
||||
|
||||
// Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
|
||||
s = s.replace(/\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“'‘«—])/g, '$1§DOT§');
|
||||
|
||||
// Step 4: Protect common honorifics, titles, and abbreviations
|
||||
const abbrPattern = /\b(Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec)\./gi;
|
||||
s = s.replace(abbrPattern, '$1§DOT§');
|
||||
|
||||
// Step 5: Protect ellipsis (... or ....)
|
||||
s = s.replace(/\.{2,}/g, (m) => '§ELLIPSIS§' + '§DOT§'.repeat(m.length));
|
||||
|
||||
// Step 6: Protect any period followed by lowercase letter (e.g. 'word. otherword')
|
||||
s = s.replace(/\.([\'"”’»)]*\s+[a-zа-яё])/g, '§DOT§$1');
|
||||
|
||||
// Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote/start of dialog
|
||||
const marked = s.replace(/([.!?]['"”’»)]*)(\s+)(?=[A-Z0-9"“'‘«—]|$)/g, '$1§SEP§');
|
||||
|
||||
return marked.split('§SEP§')
|
||||
.map(part => part.replace(/§DOT§/g, '.').replace(/§ELLIPSIS§/g, '').trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function extractContextSentence(fullText, word) {
|
||||
if (!fullText) return '';
|
||||
let protectedText = fullText;
|
||||
for (const h of READER_HONORIFICS) {
|
||||
if (protectedText.toLowerCase().includes(h.toLowerCase())) {
|
||||
const safe = h.replace(/\./g, '§DOT§');
|
||||
const reH = new RegExp(escapeRegex(h), 'gi');
|
||||
protectedText = protectedText.replace(reH, safe);
|
||||
}
|
||||
}
|
||||
// Protect decimal numbers (e.g. 3.14, 1.5)
|
||||
protectedText = protectedText.replace(/(\d+)\.(\d+)/g, '$1§DOT§$2');
|
||||
|
||||
// Split sentences on terminal punctuation (.!?) followed by optional quotes/brackets and whitespace
|
||||
const marked = protectedText.replace(/([.!?]['"”’»)]*)\s+/g, '$1§SEP§');
|
||||
const rawSentences = marked.split('§SEP§');
|
||||
|
||||
const sentences = robustSentenceSplit(fullText);
|
||||
const cleanWord = (word || '').trim();
|
||||
if (cleanWord) {
|
||||
const escaped = escapeRegex(cleanWord);
|
||||
const wordRe = new RegExp(`(?:^|\\W)${escaped}(?:$|\\W)`, 'i');
|
||||
for (const s of rawSentences) {
|
||||
for (const s of sentences) {
|
||||
if (wordRe.test(s)) {
|
||||
return s.replace(/§DOT§/g, '.').trim();
|
||||
return s.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fullText.slice(0, 160).trim();
|
||||
return sentences.length > 0 ? sentences[0].replace(/\s+/g, ' ').trim() : fullText.slice(0, 160).trim();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -1176,7 +1187,7 @@ export function positionWordPopup(targetElement) {
|
||||
readerWordPopup.style.left = `${Math.round(left)}px`;
|
||||
}
|
||||
|
||||
export async function openWordPopup(term, contextSentence, targetElement) {
|
||||
export async function openWordPopup(term, contextSentence, targetElement, paragraphContext = '') {
|
||||
const readerWordPopup = document.getElementById('reader-word-popup');
|
||||
const readerSettingsPanel = document.getElementById('reader-settings-panel');
|
||||
const popupTerm = document.getElementById('popup-term');
|
||||
@@ -1202,6 +1213,7 @@ export async function openWordPopup(term, contextSentence, targetElement) {
|
||||
popupCurrentData = {
|
||||
term: cleanTerm,
|
||||
contextSentence: contextSentence,
|
||||
paragraphContext: paragraphContext,
|
||||
targetElement: targetElement,
|
||||
existingCard: existingCard,
|
||||
};
|
||||
@@ -1339,7 +1351,7 @@ export async function openWordPopup(term, contextSentence, targetElement) {
|
||||
|
||||
export async function handleWordClick(word, paragraphText, targetElement) {
|
||||
const sentence = extractContextSentence(paragraphText, word);
|
||||
openWordPopup(word, sentence, targetElement);
|
||||
openWordPopup(word, sentence, targetElement, paragraphText);
|
||||
}
|
||||
|
||||
function updatePopupPositionOnScroll() {
|
||||
@@ -1965,7 +1977,7 @@ export function initReader() {
|
||||
if (popupContextMeaningWrap) popupContextMeaningWrap.classList.add('hidden');
|
||||
|
||||
try {
|
||||
const res = await api.lookup.translateSentence(ctxSentence, term);
|
||||
const res = await api.lookup.translateSentence(ctxSentence, term, popupCurrentData.paragraphContext || '');
|
||||
if (popupSentenceLoading) popupSentenceLoading.classList.add('hidden');
|
||||
if (popupSentenceTranslationWrap) popupSentenceTranslationWrap.classList.remove('hidden');
|
||||
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
// Service Worker for Notes English PWA (Stage 2)
|
||||
// Strategy: Cache-first for static assets, Network-first for API calls.
|
||||
|
||||
const CACHE_NAME = 'notes-v5.9';
|
||||
const CACHE_NAME = 'notes-v6.0';
|
||||
const OFFLINE_URL = '/';
|
||||
|
||||
const PRECACHE_URLS = [
|
||||
|
||||
Reference in New Issue
Block a user