Antigravity
532e2ec3b2
- Implement robustSentenceSplit across reader.js and backend (miner.py, cefr.py) to prevent false cutoffs on honorifics (Mr., Mrs., Dr.), single-letter initials (John F. Kennedy), decimals, acronyms (U.S., U.K.), and abbreviations (No. 4, etc.) - Upgrade translation prompt to professional literary Russian, eliminating literal word-for-word translation and preserving titles/honorifics - Inject surrounding paragraph context into translation API to avoid out-of-context translations - Inject offline dictionary hints into LLM prompt for accurate contextual word meaning - Prefer qwen2.5:3b for higher translation fluency and fidelity - Purge truncated cache entries from sentence_translations and restore Card 30 (briefcase) with authentic full sentence - Bump Service Worker cache to notes-v6.0 - Add unit tests for robust sentence splitting, prompt formatting, and translation API
497 lines
19 KiB
Python
497 lines
19 KiB
Python
"""
|
||
Book Sentence Mining & Automated CEFR Question Generator.
|
||
Extracts real, natural sentences from user's uploaded books (EPUB, PDF, TXT),
|
||
identifies target CEFR vocabulary, blanks out the target word,
|
||
and matches plausible distractors of the same Part of Speech.
|
||
100% offline, zero-dependency, pure Python stdlib + SQLite.
|
||
"""
|
||
|
||
import re
|
||
import json
|
||
import random
|
||
import sqlite3
|
||
import asyncio
|
||
from typing import Dict, Any, List, Optional, Tuple
|
||
|
||
from app.tasks import update_task_progress, complete_task, fail_task, is_task_cancelled
|
||
|
||
# Curated Oxford 3000/5000 benchmark lexicon categorized by level and part-of-speech
|
||
CEFR_LEXICON: Dict[str, Dict[str, List[str]]] = {
|
||
"A1": {
|
||
"adjective": [
|
||
"expensive", "cheap", "careful", "dangerous", "healthy", "boring", "famous",
|
||
"simple", "wonderful", "wrong", "clean", "heavy", "narrow", "empty",
|
||
"hungry", "thirsty", "quiet", "loud", "clever"
|
||
],
|
||
"verb": [
|
||
"remember", "decide", "invite", "borrow", "listen", "repeat", "travel",
|
||
"welcome", "laugh", "shout", "wash", "catch", "throw", "build", "climb"
|
||
],
|
||
"noun": [
|
||
"weather", "breakfast", "village", "journey", "holiday", "hospital", "library",
|
||
"message", "package", "question", "station", "ticket", "uniform", "bottle",
|
||
"island", "pocket", "mirror", "market", "bridge", "corner", "cloud", "floor",
|
||
"blanket"
|
||
],
|
||
},
|
||
"A2": {
|
||
"adjective": [
|
||
"necessary", "strange", "patient", "proud", "separate", "urgent", "wealthy",
|
||
"afraid", "complete", "connected", "disappointed", "jealous", "slow"
|
||
],
|
||
"verb": [
|
||
"discover", "improve", "protect", "remain", "suggest", "compete", "earn",
|
||
"refuse", "repair", "warn", "attend", "celebrate", "afford", "arrange",
|
||
"avoid", "compare", "contain", "damage", "depend", "describe", "discuss",
|
||
"doubt", "expect", "explain", "fail", "guess", "manage", "notice", "succeed"
|
||
],
|
||
"noun": [
|
||
"accident", "advice", "customer", "equipment", "salary", "traffic", "transport",
|
||
"benefit", "habit", "choice", "condition", "crowd", "culture", "degree",
|
||
"effect", "energy", "environment", "experience", "flight", "future", "nature",
|
||
"opinion", "reason"
|
||
],
|
||
},
|
||
"B1": {
|
||
"adjective": [
|
||
"mysterious", "ambitious", "curious", "efficient", "fluent", "frequent",
|
||
"genuine", "persistent", "reasonable", "sensible", "suitable", "temporary",
|
||
"universal", "valuable", "vulnerable", "confident", "essential", "critical",
|
||
"adequate", "apparent", "accurate"
|
||
],
|
||
"verb": [
|
||
"accomplish", "convince", "emphasize", "intend", "obtain", "predict",
|
||
"threaten", "approve", "broadcast", "cooperate", "distinguish", "eliminate",
|
||
"guarantee", "identify", "achieve", "consider", "develop", "encourage",
|
||
"hesitate", "influence", "interrupt", "maintain", "negotiate", "persuade"
|
||
],
|
||
"noun": [
|
||
"secret", "anxiety", "circumstance", "community", "decision", "mystery",
|
||
"opportunity", "relationship", "requirement", "consequence", "behavior",
|
||
"challenge"
|
||
],
|
||
},
|
||
"B2": {
|
||
"adjective": [
|
||
"meticulous", "novel", "relentless", "subtle", "tangible", "ubiquitous",
|
||
"unprecedented", "viable", "zealous", "ambiguous", "benevolent", "complacent",
|
||
"diligent", "ephemeral", "formidable", "innovative", "gregarious", "homogeneous",
|
||
"lucrative", "eligible", "arbitrary", "coherent", "comprehensive", "crucial",
|
||
"feasible", "fundamental", "inevitable", "inherent", "plausible", "preliminary",
|
||
"profound", "resilient", "simultaneous", "sophisticated", "substantial"
|
||
],
|
||
"verb": [
|
||
"deteriorate", "scrutinize", "withstand", "yield", "diminish", "undermine",
|
||
"fluctuate", "facilitate", "integrate", "deduce", "accommodate", "advocate",
|
||
"allocate", "convey", "elaborate", "enhance", "justify", "perceive"
|
||
],
|
||
"noun": [
|
||
"discrepancy", "compromise", "consensus", "ambiguity", "contradiction",
|
||
"paradox", "hypothesis"
|
||
],
|
||
},
|
||
}
|
||
|
||
# Reverse lookup index: term -> list of (level, pos)
|
||
_WORD_INDEX: Dict[str, List[Tuple[str, str]]] = {}
|
||
for _lvl, _pos_map in CEFR_LEXICON.items():
|
||
for _pos, _words in _pos_map.items():
|
||
for _w in _words:
|
||
_w_low = _w.lower()
|
||
if _w_low not in _WORD_INDEX:
|
||
_WORD_INDEX[_w_low] = []
|
||
_WORD_INDEX[_w_low].append((_lvl, _pos))
|
||
|
||
def robust_sentence_split(text: str) -> List[str]:
|
||
"""
|
||
Splits English narrative prose into clean, complete sentences.
|
||
Protects decimals, acronyms, single-letter initials, honorifics, titles, and abbreviations.
|
||
"""
|
||
if not text:
|
||
return []
|
||
|
||
# Step 1: Protect decimal numbers (e.g. 3.14, 1.5)
|
||
s = re.sub(r"(\d+)\.(\d+)", r"\1§DOT§\2", text)
|
||
|
||
# Step 2: Protect acronyms with dots like U.S., U.K., e.g., i.e., a.m., p.m., Ph.D.
|
||
s = re.sub(r"\b((?:[A-Za-z]\.){2,})", lambda m: m.group(1).replace(".", "§DOT§"), s)
|
||
|
||
# Step 3: Protect single-letter initials like 'John F. Kennedy', 'J. K. Rowling'
|
||
s = re.sub(r'\b([A-Za-z])\.(?=\s+[A-Za-z0-9"“\'‘«—])', r"\1§DOT§", s)
|
||
|
||
# Step 4: Protect common honorifics, titles, and abbreviations
|
||
abbr_pattern = (
|
||
r"\b("
|
||
r"Mr|Mrs|Ms|Miss|Dr|Prof|Rev|Hon|Capt|Col|Gen|Lt|Cmdr|Sgt|Cpl|Pvt|Maj|Adm|"
|
||
r"Gov|Sen|Rep|Pres|Sec|Treas|Insp|Det|Supt|Messrs|Mmes|St|Mt|Ft|"
|
||
r"etc|vs|v|Jr|Sr|approx|app|apt|dept|univ|assn|assoc|bldg|co|corp|inc|ltd|"
|
||
r"est|misc|al|ibid|op|cit|ed|eds|trans|vol|vols|no|nos|sec|secs|ch|chs|"
|
||
r"fig|figs|p|pp|ave|blvd|rd|st|ct|ln|dr|hwy|"
|
||
r"jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
|
||
r")\."
|
||
)
|
||
s = re.sub(abbr_pattern, r"\1§DOT§", s, flags=re.IGNORECASE)
|
||
|
||
# Step 5: Protect ellipsis (... or ....)
|
||
s = re.sub(r"\.{2,}", lambda m: "§ELLIPSIS§" + ("§DOT§" * len(m.group(0))), s)
|
||
|
||
# Step 6: Protect any period followed by lowercase letter
|
||
s = re.sub(r"\.([\'\"”’»)]*\s+[a-zа-яё])", r"§DOT§\1", s)
|
||
|
||
# Step 7: Split on sentence terminators [.!?] followed by whitespace and uppercase/number/quote
|
||
marked = re.sub(r'([.!?][\'\"”’»)]*)(\s+)(?=[A-Z0-9"“\'‘«—]|$)', r"\1§SEP§", s)
|
||
|
||
raw_sentences = [
|
||
re.sub(r"\s+", " ", part.replace("§DOT§", ".").replace("§ELLIPSIS§", "")).strip()
|
||
for part in marked.split("§SEP§")
|
||
]
|
||
return [sent for sent in raw_sentences if sent]
|
||
|
||
|
||
def split_into_clean_sentences(text: str) -> List[str]:
|
||
"""
|
||
Safely splits narrative prose into clean, grammatically complete sentences.
|
||
Protects honorifics, initials, and abbreviations from being split incorrectly.
|
||
"""
|
||
if not text:
|
||
return []
|
||
|
||
raw_splits = robust_sentence_split(text)
|
||
|
||
clean_sentences = []
|
||
for s in raw_splits:
|
||
flattened = re.sub(r"\s+", " ", s).strip()
|
||
|
||
# Basic validity filters:
|
||
# 1. Skip dialogue lines starting with quotation marks or dashes
|
||
if flattened.startswith('"') or flattened.startswith('“') or flattened.startswith('—'):
|
||
continue
|
||
# 2. Skip all-caps headers
|
||
if flattened.isupper():
|
||
continue
|
||
# 3. Must end with standard punctuation
|
||
if not re.search(r"[.!?]$", flattened):
|
||
continue
|
||
|
||
words = re.findall(r"\b[A-Za-z]+\b", flattened)
|
||
# Optimal pedagogical length: 8 to 24 words
|
||
if 8 <= len(words) <= 24:
|
||
clean_sentences.append(flattened)
|
||
|
||
return clean_sentences
|
||
|
||
|
||
def find_distractors(
|
||
conn: sqlite3.Connection,
|
||
target_term: str,
|
||
target_pos: str,
|
||
target_level: str,
|
||
count: int = 3,
|
||
) -> List[str]:
|
||
"""
|
||
Finds 3 clean, plausible distractors of the same CEFR level and part of speech.
|
||
First prioritizes words from the curated CEFR lexicon, then falls back to offline_dictionary.
|
||
"""
|
||
clean_target = target_term.lower().strip()
|
||
distractors = []
|
||
|
||
# 1. Primary pool: words from the exact same level and POS in our curated lexicon
|
||
level_dict = CEFR_LEXICON.get(target_level, {})
|
||
pool = [w.lower() for w in level_dict.get(target_pos, []) if w.lower() != clean_target]
|
||
|
||
if len(pool) >= count:
|
||
random.shuffle(pool)
|
||
distractors = pool[:count]
|
||
else:
|
||
# Expand pool with adjacent levels if needed
|
||
adjacent_levels = ["A2", "B1", "B2", "A1"]
|
||
for adj_lvl in adjacent_levels:
|
||
adj_pool = [w.lower() for w in CEFR_LEXICON.get(adj_lvl, {}).get(target_pos, []) if w.lower() != clean_target and w.lower() not in pool]
|
||
pool.extend(adj_pool)
|
||
if len(pool) >= count:
|
||
break
|
||
random.shuffle(pool)
|
||
distractors = pool[:count]
|
||
|
||
# Ensure we have required count, else query offline_dictionary
|
||
if len(distractors) < count:
|
||
pos_prefix = target_pos.capitalize()
|
||
extra_rows = conn.execute(
|
||
"""
|
||
SELECT term FROM offline_dictionary
|
||
WHERE pos LIKE ? AND length(term) BETWEEN 4 AND 12
|
||
AND term NOT LIKE '% %' AND term NOT LIKE '%-%'
|
||
AND lower(term) != ?
|
||
ORDER BY RANDOM() LIMIT 20
|
||
""",
|
||
(f"{pos_prefix}%", clean_target),
|
||
).fetchall()
|
||
for r in extra_rows:
|
||
w = r["term"].lower()
|
||
if w != clean_target and w not in distractors:
|
||
distractors.append(w)
|
||
if len(distractors) >= count:
|
||
break
|
||
|
||
return distractors[:count]
|
||
|
||
|
||
def clean_translation(trans: str) -> str:
|
||
"""Cleans up raw translation strings, removes slash phonetic residues and romanizations."""
|
||
if not trans:
|
||
return ""
|
||
t = re.sub(r"/[^/]+/?", "", trans)
|
||
parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()]
|
||
clean_parts = []
|
||
for p in parts:
|
||
if re.search(r"[а-яА-ЯёЁ]", p):
|
||
clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip()
|
||
if clean_p:
|
||
clean_parts.append(clean_p)
|
||
return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip()
|
||
|
||
|
||
async def mine_book_questions(
|
||
conn: sqlite3.Connection,
|
||
user_id: int,
|
||
book_id: Optional[int],
|
||
target_level: str,
|
||
target_count: int = 15,
|
||
task_id: Optional[int] = None,
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Asynchronously mines candidate sentences from user's book library,
|
||
identifies CEFR vocabulary targets, generates balanced multiple-choice questions,
|
||
and stores them directly into cefr_questions.
|
||
"""
|
||
clean_target_level = target_level.upper().strip() if target_level else "ALL"
|
||
|
||
# Fetch candidate books
|
||
if book_id and book_id > 0:
|
||
book_rows = conn.execute(
|
||
"SELECT id, title, chapters_json FROM books WHERE id = ? AND user_id = ?",
|
||
(book_id, user_id),
|
||
).fetchall()
|
||
else:
|
||
book_rows = conn.execute(
|
||
"SELECT id, title, chapters_json FROM books WHERE user_id = ? ORDER BY updated_at DESC",
|
||
(user_id,),
|
||
).fetchall()
|
||
|
||
if not book_rows:
|
||
if task_id:
|
||
fail_task(conn, task_id, "В вашей библиотеке не найдено книг для анализа.")
|
||
return {"status": "error", "message": "No books found"}
|
||
|
||
mined_questions: List[Dict[str, Any]] = []
|
||
mined_count = 0
|
||
scanned_sentences = 0
|
||
|
||
# Collect existing sentences already tested in cefr_questions to prevent duplicate questions
|
||
existing_sentences = set(
|
||
r[0] for r in conn.execute("SELECT sentence FROM cefr_questions").fetchall()
|
||
)
|
||
# Track terms mined in THIS session to ensure diversity of vocabulary
|
||
session_terms = set()
|
||
|
||
for book in book_rows:
|
||
b_id = book["id"]
|
||
b_title = book["title"]
|
||
try:
|
||
chapters = json.loads(book["chapters_json"] or "[]")
|
||
except Exception:
|
||
chapters = []
|
||
|
||
for ch_idx, ch in enumerate(chapters):
|
||
if is_task_cancelled(task_id, conn):
|
||
break
|
||
|
||
ch_title = ch.get("title") or f"Глава {ch_idx + 1}"
|
||
ch_content = ch.get("content", "")
|
||
sentences = split_into_clean_sentences(ch_content)
|
||
|
||
for sentence in sentences:
|
||
if is_task_cancelled(task_id, conn):
|
||
break
|
||
|
||
scanned_sentences += 1
|
||
words = re.findall(r"\b[A-Za-z]+\b", sentence)
|
||
|
||
for word in words:
|
||
w_low = word.lower()
|
||
if w_low in session_terms:
|
||
continue
|
||
|
||
# Check if word is in our target CEFR lexicon
|
||
matches = _WORD_INDEX.get(w_low, [])
|
||
if not matches:
|
||
continue
|
||
|
||
# Filter by target level if requested
|
||
selected_match = None
|
||
if clean_target_level != "ALL":
|
||
for lvl, pos in matches:
|
||
if lvl == clean_target_level:
|
||
selected_match = (lvl, pos)
|
||
break
|
||
else:
|
||
selected_match = matches[0]
|
||
|
||
if not selected_match:
|
||
continue
|
||
|
||
lvl, pos = selected_match
|
||
|
||
# Create cloze gap (only first match)
|
||
blanked_sentence = re.sub(
|
||
r"\b" + re.escape(word) + r"\b",
|
||
"___",
|
||
sentence,
|
||
count=1,
|
||
)
|
||
if blanked_sentence in existing_sentences:
|
||
continue
|
||
|
||
# Look up translation and phonetic from offline_dictionary
|
||
dict_row = conn.execute(
|
||
"SELECT translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
|
||
(w_low,),
|
||
).fetchone()
|
||
if not dict_row or not dict_row["translation"]:
|
||
continue
|
||
|
||
translation = clean_translation(dict_row["translation"])
|
||
phonetic = dict_row["phonetic"] or ""
|
||
|
||
# Find 3 suitable distractors
|
||
distractors = find_distractors(conn, w_low, pos, lvl, count=3)
|
||
if len(distractors) < 3:
|
||
continue
|
||
|
||
options = [w_low] + distractors
|
||
random.shuffle(options)
|
||
|
||
explanation = f"{word.capitalize()} ({translation}) — пример из книги «{b_title}» ({ch_title})."
|
||
|
||
# Insert into cefr_questions
|
||
try:
|
||
cursor = conn.execute(
|
||
"""
|
||
INSERT OR IGNORE INTO cefr_questions (
|
||
level, term, translation, pos, phonetic,
|
||
sentence, options_json, explanation,
|
||
source_book_id, source_book_title, source_chapter_title
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
lvl,
|
||
w_low,
|
||
translation,
|
||
pos,
|
||
phonetic,
|
||
blanked_sentence,
|
||
json.dumps(options, ensure_ascii=False),
|
||
explanation,
|
||
b_id,
|
||
b_title,
|
||
ch_title,
|
||
),
|
||
)
|
||
if cursor.rowcount > 0:
|
||
existing_sentences.add(blanked_sentence)
|
||
session_terms.add(w_low)
|
||
mined_count += 1
|
||
mined_questions.append({
|
||
"id": cursor.lastrowid,
|
||
"level": lvl,
|
||
"term": w_low,
|
||
"sentence": blanked_sentence,
|
||
"book_title": b_title,
|
||
"chapter_title": ch_title,
|
||
})
|
||
|
||
if task_id:
|
||
update_task_progress(
|
||
conn,
|
||
task_id,
|
||
mined_count,
|
||
target_count,
|
||
f"Собрано {mined_count}/{target_count} вопросов из «{b_title}»",
|
||
)
|
||
conn.commit()
|
||
|
||
# Cooperative yield
|
||
await asyncio.sleep(0)
|
||
|
||
if mined_count >= target_count:
|
||
break
|
||
except Exception:
|
||
pass
|
||
|
||
if mined_count >= target_count:
|
||
break
|
||
if mined_count >= target_count or is_task_cancelled(task_id, conn):
|
||
break
|
||
if mined_count >= target_count or is_task_cancelled(task_id, conn):
|
||
break
|
||
|
||
# Finalize task status
|
||
if task_id:
|
||
if is_task_cancelled(task_id, conn):
|
||
pass
|
||
elif mined_count > 0:
|
||
complete_task(
|
||
conn,
|
||
task_id,
|
||
result={
|
||
"mined_count": mined_count,
|
||
"target_count": target_count,
|
||
"target_level": clean_target_level,
|
||
"scanned_sentences": scanned_sentences,
|
||
},
|
||
message=f"Успешно создано {mined_count} новых вопросов из ваших книг!",
|
||
)
|
||
conn.commit()
|
||
else:
|
||
complete_task(
|
||
conn,
|
||
task_id,
|
||
result={"mined_count": 0, "scanned_sentences": scanned_sentences},
|
||
message="Все найденные слова книги уже есть в вашей базе вопросов.",
|
||
)
|
||
conn.commit()
|
||
|
||
return {
|
||
"status": "ok",
|
||
"mined_count": mined_count,
|
||
"target_count": target_count,
|
||
"questions": mined_questions,
|
||
}
|
||
|
||
|
||
async def run_mining_task_background(
|
||
task_id: int,
|
||
user_id: int,
|
||
book_id: Optional[int],
|
||
target_level: str,
|
||
target_count: int = 15,
|
||
) -> None:
|
||
"""Entry point for asyncio background task execution with isolated DB connection."""
|
||
from app.database import get_db_connection
|
||
conn = get_db_connection()
|
||
try:
|
||
await mine_book_questions(
|
||
conn=conn,
|
||
user_id=user_id,
|
||
book_id=book_id,
|
||
target_level=target_level,
|
||
target_count=target_count,
|
||
task_id=task_id,
|
||
)
|
||
except Exception as e:
|
||
fail_task(conn, task_id, f"Ошибка сборки: {str(e)}")
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|