feat: background tasks manager and book sentence mining for CEFR tests
This commit is contained in:
+462
@@ -0,0 +1,462 @@
|
||||
"""
|
||||
Book Sentence Mining & Automated CEFR Question Generator.
|
||||
Extracts real, natural sentences from user's uploaded books (EPUB, PDF, TXT),
|
||||
identifies target CEFR vocabulary, blanks out the target word,
|
||||
and matches plausible distractors of the same Part of Speech.
|
||||
100% offline, zero-dependency, pure Python stdlib + SQLite.
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
import sqlite3
|
||||
import asyncio
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
|
||||
from app.tasks import update_task_progress, complete_task, fail_task, is_task_cancelled
|
||||
|
||||
# Curated Oxford 3000/5000 benchmark lexicon categorized by level and part-of-speech
|
||||
CEFR_LEXICON: Dict[str, Dict[str, List[str]]] = {
|
||||
"A1": {
|
||||
"adjective": [
|
||||
"expensive", "cheap", "careful", "dangerous", "healthy", "boring", "famous",
|
||||
"simple", "wonderful", "wrong", "clean", "heavy", "narrow", "empty",
|
||||
"hungry", "thirsty", "quiet", "loud", "clever"
|
||||
],
|
||||
"verb": [
|
||||
"remember", "decide", "invite", "borrow", "listen", "repeat", "travel",
|
||||
"welcome", "laugh", "shout", "wash", "catch", "throw", "build", "climb"
|
||||
],
|
||||
"noun": [
|
||||
"weather", "breakfast", "village", "journey", "holiday", "hospital", "library",
|
||||
"message", "package", "question", "station", "ticket", "uniform", "bottle",
|
||||
"island", "pocket", "mirror", "market", "bridge", "corner", "cloud", "floor",
|
||||
"blanket"
|
||||
],
|
||||
},
|
||||
"A2": {
|
||||
"adjective": [
|
||||
"necessary", "strange", "patient", "proud", "separate", "urgent", "wealthy",
|
||||
"afraid", "complete", "connected", "disappointed", "jealous", "slow"
|
||||
],
|
||||
"verb": [
|
||||
"discover", "improve", "protect", "remain", "suggest", "compete", "earn",
|
||||
"refuse", "repair", "warn", "attend", "celebrate", "afford", "arrange",
|
||||
"avoid", "compare", "contain", "damage", "depend", "describe", "discuss",
|
||||
"doubt", "expect", "explain", "fail", "guess", "manage", "notice", "succeed"
|
||||
],
|
||||
"noun": [
|
||||
"accident", "advice", "customer", "equipment", "salary", "traffic", "transport",
|
||||
"benefit", "habit", "choice", "condition", "crowd", "culture", "degree",
|
||||
"effect", "energy", "environment", "experience", "flight", "future", "nature",
|
||||
"opinion", "reason"
|
||||
],
|
||||
},
|
||||
"B1": {
|
||||
"adjective": [
|
||||
"mysterious", "ambitious", "curious", "efficient", "fluent", "frequent",
|
||||
"genuine", "persistent", "reasonable", "sensible", "suitable", "temporary",
|
||||
"universal", "valuable", "vulnerable", "confident", "essential", "critical",
|
||||
"adequate", "apparent", "accurate"
|
||||
],
|
||||
"verb": [
|
||||
"accomplish", "convince", "emphasize", "intend", "obtain", "predict",
|
||||
"threaten", "approve", "broadcast", "cooperate", "distinguish", "eliminate",
|
||||
"guarantee", "identify", "achieve", "consider", "develop", "encourage",
|
||||
"hesitate", "influence", "interrupt", "maintain", "negotiate", "persuade"
|
||||
],
|
||||
"noun": [
|
||||
"secret", "anxiety", "circumstance", "community", "decision", "mystery",
|
||||
"opportunity", "relationship", "requirement", "consequence", "behavior",
|
||||
"challenge"
|
||||
],
|
||||
},
|
||||
"B2": {
|
||||
"adjective": [
|
||||
"meticulous", "novel", "relentless", "subtle", "tangible", "ubiquitous",
|
||||
"unprecedented", "viable", "zealous", "ambiguous", "benevolent", "complacent",
|
||||
"diligent", "ephemeral", "formidable", "innovative", "gregarious", "homogeneous",
|
||||
"lucrative", "eligible", "arbitrary", "coherent", "comprehensive", "crucial",
|
||||
"feasible", "fundamental", "inevitable", "inherent", "plausible", "preliminary",
|
||||
"profound", "resilient", "simultaneous", "sophisticated", "substantial"
|
||||
],
|
||||
"verb": [
|
||||
"deteriorate", "scrutinize", "withstand", "yield", "diminish", "undermine",
|
||||
"fluctuate", "facilitate", "integrate", "deduce", "accommodate", "advocate",
|
||||
"allocate", "convey", "elaborate", "enhance", "justify", "perceive"
|
||||
],
|
||||
"noun": [
|
||||
"discrepancy", "compromise", "consensus", "ambiguity", "contradiction",
|
||||
"paradox", "hypothesis"
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# Reverse lookup index: term -> list of (level, pos)
|
||||
_WORD_INDEX: Dict[str, List[Tuple[str, str]]] = {}
|
||||
for _lvl, _pos_map in CEFR_LEXICON.items():
|
||||
for _pos, _words in _pos_map.items():
|
||||
for _w in _words:
|
||||
_w_low = _w.lower()
|
||||
if _w_low not in _WORD_INDEX:
|
||||
_WORD_INDEX[_w_low] = []
|
||||
_WORD_INDEX[_w_low].append((_lvl, _pos))
|
||||
|
||||
# Common abbreviations to prevent false splits
|
||||
_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."]
|
||||
|
||||
|
||||
def split_into_clean_sentences(text: str) -> List[str]:
|
||||
"""
|
||||
Safely splits narrative prose into clean, grammatically complete sentences.
|
||||
Protects honorifics and abbreviations from being split incorrectly.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
# Protect common abbreviations and titles
|
||||
protected = text
|
||||
for abbr in _HONORIFICS:
|
||||
protected = protected.replace(abbr, abbr.replace(".", "§DOT§"))
|
||||
|
||||
# Split on sentence-ending punctuation (.!?) followed by space or newline
|
||||
raw_splits = re.split(r"(?<=[.!?])\s+", protected)
|
||||
|
||||
clean_sentences = []
|
||||
for s in raw_splits:
|
||||
restored = s.replace("§DOT§", ".").strip()
|
||||
# Flatten multiple spaces and line breaks
|
||||
flattened = re.sub(r"\s+", " ", restored)
|
||||
|
||||
# Basic validity filters:
|
||||
# 1. Skip dialogue lines starting with quotation marks or dashes
|
||||
if flattened.startswith('"') or flattened.startswith('“') or flattened.startswith('—'):
|
||||
continue
|
||||
# 2. Skip all-caps headers
|
||||
if flattened.isupper():
|
||||
continue
|
||||
# 3. Must end with standard punctuation
|
||||
if not re.search(r"[.!?]$", flattened):
|
||||
continue
|
||||
|
||||
words = re.findall(r"\b[A-Za-z]+\b", flattened)
|
||||
# Optimal pedagogical length: 8 to 24 words
|
||||
if 8 <= len(words) <= 24:
|
||||
clean_sentences.append(flattened)
|
||||
|
||||
return clean_sentences
|
||||
|
||||
|
||||
def find_distractors(
|
||||
conn: sqlite3.Connection,
|
||||
target_term: str,
|
||||
target_pos: str,
|
||||
target_level: str,
|
||||
count: int = 3,
|
||||
) -> List[str]:
|
||||
"""
|
||||
Finds 3 clean, plausible distractors of the same CEFR level and part of speech.
|
||||
First prioritizes words from the curated CEFR lexicon, then falls back to offline_dictionary.
|
||||
"""
|
||||
clean_target = target_term.lower().strip()
|
||||
distractors = []
|
||||
|
||||
# 1. Primary pool: words from the exact same level and POS in our curated lexicon
|
||||
level_dict = CEFR_LEXICON.get(target_level, {})
|
||||
pool = [w.lower() for w in level_dict.get(target_pos, []) if w.lower() != clean_target]
|
||||
|
||||
if len(pool) >= count:
|
||||
random.shuffle(pool)
|
||||
distractors = pool[:count]
|
||||
else:
|
||||
# Expand pool with adjacent levels if needed
|
||||
adjacent_levels = ["A2", "B1", "B2", "A1"]
|
||||
for adj_lvl in adjacent_levels:
|
||||
adj_pool = [w.lower() for w in CEFR_LEXICON.get(adj_lvl, {}).get(target_pos, []) if w.lower() != clean_target and w.lower() not in pool]
|
||||
pool.extend(adj_pool)
|
||||
if len(pool) >= count:
|
||||
break
|
||||
random.shuffle(pool)
|
||||
distractors = pool[:count]
|
||||
|
||||
# Ensure we have required count, else query offline_dictionary
|
||||
if len(distractors) < count:
|
||||
pos_prefix = target_pos.capitalize()
|
||||
extra_rows = conn.execute(
|
||||
"""
|
||||
SELECT term FROM offline_dictionary
|
||||
WHERE pos LIKE ? AND length(term) BETWEEN 4 AND 12
|
||||
AND term NOT LIKE '% %' AND term NOT LIKE '%-%'
|
||||
AND lower(term) != ?
|
||||
ORDER BY RANDOM() LIMIT 20
|
||||
""",
|
||||
(f"{pos_prefix}%", clean_target),
|
||||
).fetchall()
|
||||
for r in extra_rows:
|
||||
w = r["term"].lower()
|
||||
if w != clean_target and w not in distractors:
|
||||
distractors.append(w)
|
||||
if len(distractors) >= count:
|
||||
break
|
||||
|
||||
return distractors[:count]
|
||||
|
||||
|
||||
def clean_translation(trans: str) -> str:
|
||||
"""Cleans up raw translation strings, removes slash phonetic residues and romanizations."""
|
||||
if not trans:
|
||||
return ""
|
||||
t = re.sub(r"/[^/]+/?", "", trans)
|
||||
parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()]
|
||||
clean_parts = []
|
||||
for p in parts:
|
||||
if re.search(r"[а-яА-ЯёЁ]", p):
|
||||
clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip()
|
||||
if clean_p:
|
||||
clean_parts.append(clean_p)
|
||||
return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip()
|
||||
|
||||
|
||||
async def mine_book_questions(
|
||||
conn: sqlite3.Connection,
|
||||
user_id: int,
|
||||
book_id: Optional[int],
|
||||
target_level: str,
|
||||
target_count: int = 15,
|
||||
task_id: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Asynchronously mines candidate sentences from user's book library,
|
||||
identifies CEFR vocabulary targets, generates balanced multiple-choice questions,
|
||||
and stores them directly into cefr_questions.
|
||||
"""
|
||||
clean_target_level = target_level.upper().strip() if target_level else "ALL"
|
||||
|
||||
# Fetch candidate books
|
||||
if book_id and book_id > 0:
|
||||
book_rows = conn.execute(
|
||||
"SELECT id, title, chapters_json FROM books WHERE id = ? AND user_id = ?",
|
||||
(book_id, user_id),
|
||||
).fetchall()
|
||||
else:
|
||||
book_rows = conn.execute(
|
||||
"SELECT id, title, chapters_json FROM books WHERE user_id = ? ORDER BY updated_at DESC",
|
||||
(user_id,),
|
||||
).fetchall()
|
||||
|
||||
if not book_rows:
|
||||
if task_id:
|
||||
fail_task(conn, task_id, "В вашей библиотеке не найдено книг для анализа.")
|
||||
return {"status": "error", "message": "No books found"}
|
||||
|
||||
mined_questions: List[Dict[str, Any]] = []
|
||||
mined_count = 0
|
||||
scanned_sentences = 0
|
||||
|
||||
# Collect existing sentences already tested in cefr_questions to prevent duplicate questions
|
||||
existing_sentences = set(
|
||||
r[0] for r in conn.execute("SELECT sentence FROM cefr_questions").fetchall()
|
||||
)
|
||||
# Track terms mined in THIS session to ensure diversity of vocabulary
|
||||
session_terms = set()
|
||||
|
||||
for book in book_rows:
|
||||
b_id = book["id"]
|
||||
b_title = book["title"]
|
||||
try:
|
||||
chapters = json.loads(book["chapters_json"] or "[]")
|
||||
except Exception:
|
||||
chapters = []
|
||||
|
||||
for ch_idx, ch in enumerate(chapters):
|
||||
if is_task_cancelled(task_id, conn):
|
||||
break
|
||||
|
||||
ch_title = ch.get("title") or f"Глава {ch_idx + 1}"
|
||||
ch_content = ch.get("content", "")
|
||||
sentences = split_into_clean_sentences(ch_content)
|
||||
|
||||
for sentence in sentences:
|
||||
if is_task_cancelled(task_id, conn):
|
||||
break
|
||||
|
||||
scanned_sentences += 1
|
||||
words = re.findall(r"\b[A-Za-z]+\b", sentence)
|
||||
|
||||
for word in words:
|
||||
w_low = word.lower()
|
||||
if w_low in session_terms:
|
||||
continue
|
||||
|
||||
# Check if word is in our target CEFR lexicon
|
||||
matches = _WORD_INDEX.get(w_low, [])
|
||||
if not matches:
|
||||
continue
|
||||
|
||||
# Filter by target level if requested
|
||||
selected_match = None
|
||||
if clean_target_level != "ALL":
|
||||
for lvl, pos in matches:
|
||||
if lvl == clean_target_level:
|
||||
selected_match = (lvl, pos)
|
||||
break
|
||||
else:
|
||||
selected_match = matches[0]
|
||||
|
||||
if not selected_match:
|
||||
continue
|
||||
|
||||
lvl, pos = selected_match
|
||||
|
||||
# Create cloze gap (only first match)
|
||||
blanked_sentence = re.sub(
|
||||
r"\b" + re.escape(word) + r"\b",
|
||||
"___",
|
||||
sentence,
|
||||
count=1,
|
||||
)
|
||||
if blanked_sentence in existing_sentences:
|
||||
continue
|
||||
|
||||
# Look up translation and phonetic from offline_dictionary
|
||||
dict_row = conn.execute(
|
||||
"SELECT translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
|
||||
(w_low,),
|
||||
).fetchone()
|
||||
if not dict_row or not dict_row["translation"]:
|
||||
continue
|
||||
|
||||
translation = clean_translation(dict_row["translation"])
|
||||
phonetic = dict_row["phonetic"] or ""
|
||||
|
||||
# Find 3 suitable distractors
|
||||
distractors = find_distractors(conn, w_low, pos, lvl, count=3)
|
||||
if len(distractors) < 3:
|
||||
continue
|
||||
|
||||
options = [w_low] + distractors
|
||||
random.shuffle(options)
|
||||
|
||||
explanation = f"{word.capitalize()} ({translation}) — пример из книги «{b_title}» ({ch_title})."
|
||||
|
||||
# Insert into cefr_questions
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO cefr_questions (
|
||||
level, term, translation, pos, phonetic,
|
||||
sentence, options_json, explanation,
|
||||
source_book_id, source_book_title, source_chapter_title
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
lvl,
|
||||
w_low,
|
||||
translation,
|
||||
pos,
|
||||
phonetic,
|
||||
blanked_sentence,
|
||||
json.dumps(options, ensure_ascii=False),
|
||||
explanation,
|
||||
b_id,
|
||||
b_title,
|
||||
ch_title,
|
||||
),
|
||||
)
|
||||
if cursor.rowcount > 0:
|
||||
existing_sentences.add(blanked_sentence)
|
||||
session_terms.add(w_low)
|
||||
mined_count += 1
|
||||
mined_questions.append({
|
||||
"id": cursor.lastrowid,
|
||||
"level": lvl,
|
||||
"term": w_low,
|
||||
"sentence": blanked_sentence,
|
||||
"book_title": b_title,
|
||||
"chapter_title": ch_title,
|
||||
})
|
||||
|
||||
if task_id:
|
||||
update_task_progress(
|
||||
conn,
|
||||
task_id,
|
||||
mined_count,
|
||||
target_count,
|
||||
f"Собрано {mined_count}/{target_count} вопросов из «{b_title}»",
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# Cooperative yield
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if mined_count >= target_count:
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if mined_count >= target_count:
|
||||
break
|
||||
if mined_count >= target_count or is_task_cancelled(task_id, conn):
|
||||
break
|
||||
if mined_count >= target_count or is_task_cancelled(task_id, conn):
|
||||
break
|
||||
|
||||
# Finalize task status
|
||||
if task_id:
|
||||
if is_task_cancelled(task_id, conn):
|
||||
pass
|
||||
elif mined_count > 0:
|
||||
complete_task(
|
||||
conn,
|
||||
task_id,
|
||||
result={
|
||||
"mined_count": mined_count,
|
||||
"target_count": target_count,
|
||||
"target_level": clean_target_level,
|
||||
"scanned_sentences": scanned_sentences,
|
||||
},
|
||||
message=f"Успешно создано {mined_count} новых вопросов из ваших книг!",
|
||||
)
|
||||
conn.commit()
|
||||
else:
|
||||
complete_task(
|
||||
conn,
|
||||
task_id,
|
||||
result={"mined_count": 0, "scanned_sentences": scanned_sentences},
|
||||
message="Все найденные слова книги уже есть в вашей базе вопросов.",
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"mined_count": mined_count,
|
||||
"target_count": target_count,
|
||||
"questions": mined_questions,
|
||||
}
|
||||
|
||||
|
||||
async def run_mining_task_background(
|
||||
task_id: int,
|
||||
user_id: int,
|
||||
book_id: Optional[int],
|
||||
target_level: str,
|
||||
target_count: int = 15,
|
||||
) -> None:
|
||||
"""Entry point for asyncio background task execution with isolated DB connection."""
|
||||
from app.database import get_db_connection
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
await mine_book_questions(
|
||||
conn=conn,
|
||||
user_id=user_id,
|
||||
book_id=book_id,
|
||||
target_level=target_level,
|
||||
target_count=target_count,
|
||||
task_id=task_id,
|
||||
)
|
||||
except Exception as e:
|
||||
fail_task(conn, task_id, f"Ошибка сборки: {str(e)}")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
Reference in New Issue
Block a user