diff --git a/app/cefr.py b/app/cefr.py index 68eaf60..6bdb085 100644 --- a/app/cefr.py +++ b/app/cefr.py @@ -1968,7 +1968,8 @@ def get_cefr_quiz_batch( if mode == "mistakes": rows = conn.execute( """ - SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json + SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json, + q.source_book_id, q.source_book_title, q.source_chapter_title FROM cefr_questions q JOIN cefr_progress p ON q.id = p.question_id WHERE p.user_id = ? AND q.level = ? AND p.is_mastered = 0 AND p.mistakes > 0 @@ -1981,13 +1982,14 @@ def get_cefr_quiz_batch( # 1. Unattempted questions rows = conn.execute( """ - SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json + SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json, + q.source_book_id, q.source_book_title, q.source_chapter_title FROM cefr_questions q LEFT JOIN cefr_progress p ON q.id = p.question_id AND p.user_id = ? WHERE q.level = ? AND (p.question_id IS NULL OR p.is_mastered = 0) ORDER BY CASE WHEN p.question_id IS NULL THEN 0 ELSE 1 END, p.last_tested_at ASC, - q.id ASC + q.id DESC LIMIT ? """, (user_id, clean_level, count), @@ -2000,10 +2002,11 @@ def get_cefr_quiz_batch( placeholders = ",".join(["?"] * len(existing_ids)) if existing_ids else "-1" extra = conn.execute( f""" - SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json + SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json, + q.source_book_id, q.source_book_title, q.source_chapter_title FROM cefr_questions q WHERE q.level = ? AND q.id NOT IN ({placeholders}) - ORDER BY q.id ASC + ORDER BY q.id DESC LIMIT ? """, [clean_level] + existing_ids + [needed], @@ -2020,14 +2023,18 @@ def get_cefr_quiz_batch( shuffled_options = list(options) random.shuffle(shuffled_options) - quiz_items.append({ + item = { "id": r["id"], "level": r["level"], "sentence": r["sentence"], "clue": r["translation"], "pos": r["pos"] or "", "options": shuffled_options, - }) + } + if "source_book_title" in r.keys() and r["source_book_title"]: + item["source_book_title"] = r["source_book_title"] + item["source_chapter_title"] = r["source_chapter_title"] or "" + quiz_items.append(item) return quiz_items @@ -2212,6 +2219,8 @@ def process_cefr_answer( "selected_card_id": selected_card_info["card_id"] if selected_card_info else None, "selected_card_term": selected_clean if not is_correct else None, "selected_translation": selected_card_info["translation"] if selected_card_info else None, + "source_book_title": q_row["source_book_title"] if "source_book_title" in q_row.keys() else None, + "source_chapter_title": q_row["source_chapter_title"] if "source_chapter_title" in q_row.keys() else None, "options_breakdown": options_breakdown, } diff --git a/app/database.py b/app/database.py index b2b1afe..0f3a744 100644 --- a/app/database.py +++ b/app/database.py @@ -216,12 +216,71 @@ def _migrate(conn: sqlite3.Connection): explanation TEXT DEFAULT '', frequency_rank INTEGER DEFAULT 0, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(level, term COLLATE NOCASE) + source_book_id INTEGER DEFAULT NULL, + source_book_title TEXT DEFAULT '', + source_chapter_title TEXT DEFAULT '' ); """) conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);") conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);") + # Safe migration: if old table had UNIQUE(level, term), migrate to allow rich book-mined sentences + cefr_ddl_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='cefr_questions'" + ).fetchone() + if cefr_ddl_row and "UNIQUE(level, term" in cefr_ddl_row[0]: + conn.execute("PRAGMA foreign_keys = OFF;") + conn.execute(""" + CREATE TABLE cefr_questions_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + level TEXT NOT NULL, + term TEXT NOT NULL COLLATE NOCASE, + translation TEXT NOT NULL, + pos TEXT DEFAULT '', + phonetic TEXT DEFAULT '', + sentence TEXT NOT NULL, + options_json TEXT NOT NULL, + explanation TEXT DEFAULT '', + frequency_rank INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + source_book_id INTEGER DEFAULT NULL, + source_book_title TEXT DEFAULT '', + source_chapter_title TEXT DEFAULT '' + ); + """) + conn.execute(""" + INSERT INTO cefr_questions_new ( + id, level, term, translation, pos, phonetic, sentence, options_json, + explanation, frequency_rank, created_at, source_book_id, source_book_title, source_chapter_title + ) + SELECT id, level, term, translation, pos, phonetic, sentence, options_json, + explanation, frequency_rank, created_at, + COALESCE(source_book_id, NULL), + COALESCE(source_book_title, ''), + COALESCE(source_chapter_title, '') + FROM cefr_questions; + """) + conn.execute("DROP TABLE cefr_questions;") + conn.execute("ALTER TABLE cefr_questions_new RENAME TO cefr_questions;") + conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);") + conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);") + conn.execute("PRAGMA foreign_keys = ON;") + + # Safe migration: add book source metadata columns if not present + existing_cefr_cols = { + row[1] for row in conn.execute("PRAGMA table_info(cefr_questions)").fetchall() + } + cefr_migrations = [ + ("source_book_id", "INTEGER DEFAULT NULL"), + ("source_book_title", "TEXT DEFAULT ''"), + ("source_chapter_title", "TEXT DEFAULT ''"), + ] + for col, definition in cefr_migrations: + if col not in existing_cefr_cols: + conn.execute(f"ALTER TABLE cefr_questions ADD COLUMN {col} {definition}") + + conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_cefr_q_level_sentence ON cefr_questions(level, sentence);") + conn.execute(""" CREATE TABLE IF NOT EXISTS cefr_progress ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -299,6 +358,27 @@ def _migrate(conn: sqlite3.Connection): except Exception: pass + # Ensure background_tasks table exists (Step 1: Background Tasks Engine) + conn.execute(""" + CREATE TABLE IF NOT EXISTS background_tasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + task_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + progress_current INTEGER DEFAULT 0, + progress_total INTEGER DEFAULT 0, + message TEXT DEFAULT '', + params_json TEXT DEFAULT '{}', + result_json TEXT DEFAULT '{}', + error_message TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ); + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);") + conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);") + def _seed_offline_dictionary(conn: sqlite3.Connection): diff --git a/app/main.py b/app/main.py index 074bb32..cd445b7 100644 --- a/app/main.py +++ b/app/main.py @@ -12,6 +12,8 @@ from fastapi.middleware.gzip import GZipMiddleware from pydantic import BaseModel, Field import httpx +import asyncio + from app import config from app.database import init_db, get_db from app.auth import ( @@ -45,6 +47,14 @@ from app.grammar import ( record_grammar_completion, update_handwritten_status, ) +from app.tasks import ( + create_task, + get_task_status, + get_active_task, + list_user_tasks, + cancel_task, +) +from app.miner import run_mining_task_background @@ -210,6 +220,12 @@ class GrammarHandwrittenRequest(BaseModel): handwritten: bool +class MineBooksRequest(BaseModel): + book_id: Optional[int] = None + level: str = "A1" + count: int = Field(15, ge=1, le=50) + + @@ -1557,3 +1573,103 @@ async def api_grammar_handwritten( return update_handwritten_status(conn, user["id"], req.unit_id, req.handwritten) +# --------------------------------------------------------------------------- +# Background Tasks & Book Sentence Mining Endpoints +# --------------------------------------------------------------------------- + +@app.post("/api/tasks/mine-books") +async def api_tasks_mine_books( + req: MineBooksRequest, + user: dict = Depends(require_user), +): + """ + Starts an asynchronous background task to mine CEFR Cloze Tap questions + from user's uploaded books (EPUB, TXT, PDF). + """ + level = req.level.upper().strip() + if level not in ["A1", "A2", "B1", "B2", "ALL"]: + raise HTTPException(status_code=400, detail="Invalid CEFR level") + + count = max(1, min(req.count, 50)) + + with get_db() as conn: + active = get_active_task(conn, user["id"], task_type="mine_books") + if active: + return { + "status": "already_running", + "message": "Сборка уже выполняется в фоне", + "task": active, + } + + task_id = create_task( + conn, + user_id=user["id"], + task_type="mine_books", + params={ + "book_id": req.book_id, + "level": level, + "count": count, + }, + total=count, + ) + + # Launch background asyncio task + asyncio.create_task( + run_mining_task_background( + task_id=task_id, + user_id=user["id"], + book_id=req.book_id, + target_level=level, + target_count=count, + ) + ) + + return { + "status": "ok", + "task_id": task_id, + "message": f"Запущена фоновая сборка тестов ({level})", + } + + +@app.get("/api/tasks/active") +async def api_tasks_active( + task_type: Optional[str] = Query(None), + user: dict = Depends(require_user), +): + """ + Returns current user's active background task (if any). + """ + with get_db() as conn: + task = get_active_task(conn, user["id"], task_type=task_type) + return {"task": task} + + +@app.get("/api/tasks/{task_id}") +async def api_tasks_status( + task_id: int, + user: dict = Depends(require_user), +): + """ + Returns status and progress of a background task. + """ + with get_db() as conn: + task = get_task_status(conn, task_id, user_id=user["id"]) + if not task: + raise HTTPException(status_code=404, detail="Task not found") + return {"task": task} + + +@app.post("/api/tasks/{task_id}/cancel") +async def api_tasks_cancel( + task_id: int, + user: dict = Depends(require_user), +): + """ + Cancels a running background task. + """ + with get_db() as conn: + cancelled = cancel_task(conn, task_id, user_id=user["id"]) + return {"status": "ok" if cancelled else "not_found", "cancelled": cancelled} + + + diff --git a/app/miner.py b/app/miner.py new file mode 100644 index 0000000..39d04a4 --- /dev/null +++ b/app/miner.py @@ -0,0 +1,462 @@ +""" +Book Sentence Mining & Automated CEFR Question Generator. +Extracts real, natural sentences from user's uploaded books (EPUB, PDF, TXT), +identifies target CEFR vocabulary, blanks out the target word, +and matches plausible distractors of the same Part of Speech. +100% offline, zero-dependency, pure Python stdlib + SQLite. +""" + +import re +import json +import random +import sqlite3 +import asyncio +from typing import Dict, Any, List, Optional, Tuple + +from app.tasks import update_task_progress, complete_task, fail_task, is_task_cancelled + +# Curated Oxford 3000/5000 benchmark lexicon categorized by level and part-of-speech +CEFR_LEXICON: Dict[str, Dict[str, List[str]]] = { + "A1": { + "adjective": [ + "expensive", "cheap", "careful", "dangerous", "healthy", "boring", "famous", + "simple", "wonderful", "wrong", "clean", "heavy", "narrow", "empty", + "hungry", "thirsty", "quiet", "loud", "clever" + ], + "verb": [ + "remember", "decide", "invite", "borrow", "listen", "repeat", "travel", + "welcome", "laugh", "shout", "wash", "catch", "throw", "build", "climb" + ], + "noun": [ + "weather", "breakfast", "village", "journey", "holiday", "hospital", "library", + "message", "package", "question", "station", "ticket", "uniform", "bottle", + "island", "pocket", "mirror", "market", "bridge", "corner", "cloud", "floor", + "blanket" + ], + }, + "A2": { + "adjective": [ + "necessary", "strange", "patient", "proud", "separate", "urgent", "wealthy", + "afraid", "complete", "connected", "disappointed", "jealous", "slow" + ], + "verb": [ + "discover", "improve", "protect", "remain", "suggest", "compete", "earn", + "refuse", "repair", "warn", "attend", "celebrate", "afford", "arrange", + "avoid", "compare", "contain", "damage", "depend", "describe", "discuss", + "doubt", "expect", "explain", "fail", "guess", "manage", "notice", "succeed" + ], + "noun": [ + "accident", "advice", "customer", "equipment", "salary", "traffic", "transport", + "benefit", "habit", "choice", "condition", "crowd", "culture", "degree", + "effect", "energy", "environment", "experience", "flight", "future", "nature", + "opinion", "reason" + ], + }, + "B1": { + "adjective": [ + "mysterious", "ambitious", "curious", "efficient", "fluent", "frequent", + "genuine", "persistent", "reasonable", "sensible", "suitable", "temporary", + "universal", "valuable", "vulnerable", "confident", "essential", "critical", + "adequate", "apparent", "accurate" + ], + "verb": [ + "accomplish", "convince", "emphasize", "intend", "obtain", "predict", + "threaten", "approve", "broadcast", "cooperate", "distinguish", "eliminate", + "guarantee", "identify", "achieve", "consider", "develop", "encourage", + "hesitate", "influence", "interrupt", "maintain", "negotiate", "persuade" + ], + "noun": [ + "secret", "anxiety", "circumstance", "community", "decision", "mystery", + "opportunity", "relationship", "requirement", "consequence", "behavior", + "challenge" + ], + }, + "B2": { + "adjective": [ + "meticulous", "novel", "relentless", "subtle", "tangible", "ubiquitous", + "unprecedented", "viable", "zealous", "ambiguous", "benevolent", "complacent", + "diligent", "ephemeral", "formidable", "innovative", "gregarious", "homogeneous", + "lucrative", "eligible", "arbitrary", "coherent", "comprehensive", "crucial", + "feasible", "fundamental", "inevitable", "inherent", "plausible", "preliminary", + "profound", "resilient", "simultaneous", "sophisticated", "substantial" + ], + "verb": [ + "deteriorate", "scrutinize", "withstand", "yield", "diminish", "undermine", + "fluctuate", "facilitate", "integrate", "deduce", "accommodate", "advocate", + "allocate", "convey", "elaborate", "enhance", "justify", "perceive" + ], + "noun": [ + "discrepancy", "compromise", "consensus", "ambiguity", "contradiction", + "paradox", "hypothesis" + ], + }, +} + +# Reverse lookup index: term -> list of (level, pos) +_WORD_INDEX: Dict[str, List[Tuple[str, str]]] = {} +for _lvl, _pos_map in CEFR_LEXICON.items(): + for _pos, _words in _pos_map.items(): + for _w in _words: + _w_low = _w.lower() + if _w_low not in _WORD_INDEX: + _WORD_INDEX[_w_low] = [] + _WORD_INDEX[_w_low].append((_lvl, _pos)) + +# Common abbreviations to prevent false splits +_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."] + + +def split_into_clean_sentences(text: str) -> List[str]: + """ + Safely splits narrative prose into clean, grammatically complete sentences. + Protects honorifics and abbreviations from being split incorrectly. + """ + if not text: + return [] + + # Protect common abbreviations and titles + protected = text + for abbr in _HONORIFICS: + protected = protected.replace(abbr, abbr.replace(".", "§DOT§")) + + # Split on sentence-ending punctuation (.!?) followed by space or newline + raw_splits = re.split(r"(?<=[.!?])\s+", protected) + + clean_sentences = [] + for s in raw_splits: + restored = s.replace("§DOT§", ".").strip() + # Flatten multiple spaces and line breaks + flattened = re.sub(r"\s+", " ", restored) + + # Basic validity filters: + # 1. Skip dialogue lines starting with quotation marks or dashes + if flattened.startswith('"') or flattened.startswith('“') or flattened.startswith('—'): + continue + # 2. Skip all-caps headers + if flattened.isupper(): + continue + # 3. Must end with standard punctuation + if not re.search(r"[.!?]$", flattened): + continue + + words = re.findall(r"\b[A-Za-z]+\b", flattened) + # Optimal pedagogical length: 8 to 24 words + if 8 <= len(words) <= 24: + clean_sentences.append(flattened) + + return clean_sentences + + +def find_distractors( + conn: sqlite3.Connection, + target_term: str, + target_pos: str, + target_level: str, + count: int = 3, +) -> List[str]: + """ + Finds 3 clean, plausible distractors of the same CEFR level and part of speech. + First prioritizes words from the curated CEFR lexicon, then falls back to offline_dictionary. + """ + clean_target = target_term.lower().strip() + distractors = [] + + # 1. Primary pool: words from the exact same level and POS in our curated lexicon + level_dict = CEFR_LEXICON.get(target_level, {}) + pool = [w.lower() for w in level_dict.get(target_pos, []) if w.lower() != clean_target] + + if len(pool) >= count: + random.shuffle(pool) + distractors = pool[:count] + else: + # Expand pool with adjacent levels if needed + adjacent_levels = ["A2", "B1", "B2", "A1"] + for adj_lvl in adjacent_levels: + adj_pool = [w.lower() for w in CEFR_LEXICON.get(adj_lvl, {}).get(target_pos, []) if w.lower() != clean_target and w.lower() not in pool] + pool.extend(adj_pool) + if len(pool) >= count: + break + random.shuffle(pool) + distractors = pool[:count] + + # Ensure we have required count, else query offline_dictionary + if len(distractors) < count: + pos_prefix = target_pos.capitalize() + extra_rows = conn.execute( + """ + SELECT term FROM offline_dictionary + WHERE pos LIKE ? AND length(term) BETWEEN 4 AND 12 + AND term NOT LIKE '% %' AND term NOT LIKE '%-%' + AND lower(term) != ? + ORDER BY RANDOM() LIMIT 20 + """, + (f"{pos_prefix}%", clean_target), + ).fetchall() + for r in extra_rows: + w = r["term"].lower() + if w != clean_target and w not in distractors: + distractors.append(w) + if len(distractors) >= count: + break + + return distractors[:count] + + +def clean_translation(trans: str) -> str: + """Cleans up raw translation strings, removes slash phonetic residues and romanizations.""" + if not trans: + return "" + t = re.sub(r"/[^/]+/?", "", trans) + parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()] + clean_parts = [] + for p in parts: + if re.search(r"[а-яА-ЯёЁ]", p): + clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip() + if clean_p: + clean_parts.append(clean_p) + return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip() + + +async def mine_book_questions( + conn: sqlite3.Connection, + user_id: int, + book_id: Optional[int], + target_level: str, + target_count: int = 15, + task_id: Optional[int] = None, +) -> Dict[str, Any]: + """ + Asynchronously mines candidate sentences from user's book library, + identifies CEFR vocabulary targets, generates balanced multiple-choice questions, + and stores them directly into cefr_questions. + """ + clean_target_level = target_level.upper().strip() if target_level else "ALL" + + # Fetch candidate books + if book_id and book_id > 0: + book_rows = conn.execute( + "SELECT id, title, chapters_json FROM books WHERE id = ? AND user_id = ?", + (book_id, user_id), + ).fetchall() + else: + book_rows = conn.execute( + "SELECT id, title, chapters_json FROM books WHERE user_id = ? ORDER BY updated_at DESC", + (user_id,), + ).fetchall() + + if not book_rows: + if task_id: + fail_task(conn, task_id, "В вашей библиотеке не найдено книг для анализа.") + return {"status": "error", "message": "No books found"} + + mined_questions: List[Dict[str, Any]] = [] + mined_count = 0 + scanned_sentences = 0 + + # Collect existing sentences already tested in cefr_questions to prevent duplicate questions + existing_sentences = set( + r[0] for r in conn.execute("SELECT sentence FROM cefr_questions").fetchall() + ) + # Track terms mined in THIS session to ensure diversity of vocabulary + session_terms = set() + + for book in book_rows: + b_id = book["id"] + b_title = book["title"] + try: + chapters = json.loads(book["chapters_json"] or "[]") + except Exception: + chapters = [] + + for ch_idx, ch in enumerate(chapters): + if is_task_cancelled(task_id, conn): + break + + ch_title = ch.get("title") or f"Глава {ch_idx + 1}" + ch_content = ch.get("content", "") + sentences = split_into_clean_sentences(ch_content) + + for sentence in sentences: + if is_task_cancelled(task_id, conn): + break + + scanned_sentences += 1 + words = re.findall(r"\b[A-Za-z]+\b", sentence) + + for word in words: + w_low = word.lower() + if w_low in session_terms: + continue + + # Check if word is in our target CEFR lexicon + matches = _WORD_INDEX.get(w_low, []) + if not matches: + continue + + # Filter by target level if requested + selected_match = None + if clean_target_level != "ALL": + for lvl, pos in matches: + if lvl == clean_target_level: + selected_match = (lvl, pos) + break + else: + selected_match = matches[0] + + if not selected_match: + continue + + lvl, pos = selected_match + + # Create cloze gap (only first match) + blanked_sentence = re.sub( + r"\b" + re.escape(word) + r"\b", + "___", + sentence, + count=1, + ) + if blanked_sentence in existing_sentences: + continue + + # Look up translation and phonetic from offline_dictionary + dict_row = conn.execute( + "SELECT translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", + (w_low,), + ).fetchone() + if not dict_row or not dict_row["translation"]: + continue + + translation = clean_translation(dict_row["translation"]) + phonetic = dict_row["phonetic"] or "" + + # Find 3 suitable distractors + distractors = find_distractors(conn, w_low, pos, lvl, count=3) + if len(distractors) < 3: + continue + + options = [w_low] + distractors + random.shuffle(options) + + explanation = f"{word.capitalize()} ({translation}) — пример из книги «{b_title}» ({ch_title})." + + # Insert into cefr_questions + try: + cursor = conn.execute( + """ + INSERT OR IGNORE INTO cefr_questions ( + level, term, translation, pos, phonetic, + sentence, options_json, explanation, + source_book_id, source_book_title, source_chapter_title + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + lvl, + w_low, + translation, + pos, + phonetic, + blanked_sentence, + json.dumps(options, ensure_ascii=False), + explanation, + b_id, + b_title, + ch_title, + ), + ) + if cursor.rowcount > 0: + existing_sentences.add(blanked_sentence) + session_terms.add(w_low) + mined_count += 1 + mined_questions.append({ + "id": cursor.lastrowid, + "level": lvl, + "term": w_low, + "sentence": blanked_sentence, + "book_title": b_title, + "chapter_title": ch_title, + }) + + if task_id: + update_task_progress( + conn, + task_id, + mined_count, + target_count, + f"Собрано {mined_count}/{target_count} вопросов из «{b_title}»", + ) + conn.commit() + + # Cooperative yield + await asyncio.sleep(0) + + if mined_count >= target_count: + break + except Exception: + pass + + if mined_count >= target_count: + break + if mined_count >= target_count or is_task_cancelled(task_id, conn): + break + if mined_count >= target_count or is_task_cancelled(task_id, conn): + break + + # Finalize task status + if task_id: + if is_task_cancelled(task_id, conn): + pass + elif mined_count > 0: + complete_task( + conn, + task_id, + result={ + "mined_count": mined_count, + "target_count": target_count, + "target_level": clean_target_level, + "scanned_sentences": scanned_sentences, + }, + message=f"Успешно создано {mined_count} новых вопросов из ваших книг!", + ) + conn.commit() + else: + complete_task( + conn, + task_id, + result={"mined_count": 0, "scanned_sentences": scanned_sentences}, + message="Все найденные слова книги уже есть в вашей базе вопросов.", + ) + conn.commit() + + return { + "status": "ok", + "mined_count": mined_count, + "target_count": target_count, + "questions": mined_questions, + } + + +async def run_mining_task_background( + task_id: int, + user_id: int, + book_id: Optional[int], + target_level: str, + target_count: int = 15, +) -> None: + """Entry point for asyncio background task execution with isolated DB connection.""" + from app.database import get_db_connection + conn = get_db_connection() + try: + await mine_book_questions( + conn=conn, + user_id=user_id, + book_id=book_id, + target_level=target_level, + target_count=target_count, + task_id=task_id, + ) + except Exception as e: + fail_task(conn, task_id, f"Ошибка сборки: {str(e)}") + conn.commit() + finally: + conn.close() + diff --git a/app/static/css/style.css b/app/static/css/style.css index 85d9fbd..e59a394 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -4276,7 +4276,89 @@ body { } .cefr-header { - margin-bottom: 28px; + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 16px; + margin-bottom: 24px; + flex-wrap: wrap; +} + +.cefr-header-actions { + display: flex; + align-items: center; + gap: 10px; +} + +/* Background Task Banner in CEFR Overview */ +.cefr-task-banner { + background: linear-gradient(135deg, rgba(59, 130, 246, 0.12), rgba(16, 185, 129, 0.08)); + border: 1px solid rgba(59, 130, 246, 0.3); + border-radius: var(--radius-md); + padding: 14px 18px; + margin-bottom: 24px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.08); +} + +.task-banner-inner { + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; +} + +.task-banner-content { + flex: 1; +} + +.task-banner-title { + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + color: var(--text-main); + margin-bottom: 4px; +} + +.task-banner-counts { + font-size: 12px; + color: var(--text-muted); + font-weight: 500; + margin-left: auto; +} + +.task-banner-msg { + font-size: 13px; + color: var(--text-muted); +} + +/* Book Source Badge in Question Card & Feedback */ +.cefr-book-badge { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 12px; + font-weight: 600; + color: #3b82f6; + background: rgba(59, 130, 246, 0.1); + border: 1px solid rgba(59, 130, 246, 0.25); + padding: 2px 10px; + border-radius: var(--radius-sm); + max-width: 280px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.fb-source-citation { + font-size: 12.5px; + font-weight: 500; + color: #3b82f6; + background: rgba(59, 130, 246, 0.08); + border-left: 3px solid #3b82f6; + padding: 6px 10px; + border-radius: 0 var(--radius-sm) var(--radius-sm) 0; + margin: 8px 0; } .cefr-title-wrap h2 { diff --git a/app/static/index.html b/app/static/index.html index 77cfcdb..1d8da05 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -5,7 +5,7 @@ Notes — Персональный тренажер и словарь английского языка - + @@ -608,8 +608,16 @@

🎯 Уровневые тесты CEFR (A1 – B2)

Эталонные тесты по международной шкале Oxford 3000/5000. Ошибки автоматически добавляются в ваш личный тренажёр SRS со статусом «В процессе»!

+
+ +
+ + +
@@ -1226,6 +1234,52 @@ + + + + ${res.source_book_title ? `
📖 Пример из книги «${escapeHtml(res.source_book_title)}»${res.source_chapter_title ? ` (${escapeHtml(res.source_chapter_title)})` : ''}
` : ''} ${res.explanation ? `

${escapeHtml(res.explanation)}

` : ''}
diff --git a/app/static/sw.js b/app/static/sw.js index 488a88d..f516b6c 100644 --- a/app/static/sw.js +++ b/app/static/sw.js @@ -1,7 +1,7 @@ // Service Worker for Notes English PWA (Stage 2) // Strategy: Cache-first for static assets, Network-first for API calls. -const CACHE_NAME = 'notes-v5.4'; +const CACHE_NAME = 'notes-v5.5'; const OFFLINE_URL = '/'; const PRECACHE_URLS = [ diff --git a/app/tasks.py b/app/tasks.py new file mode 100644 index 0000000..b3d418b --- /dev/null +++ b/app/tasks.py @@ -0,0 +1,295 @@ +""" +Background Tasks & Async Job Manager. +100% pure Python stdlib + SQLite WAL mode. +Zero Redis, zero Celery, zero bloat (Ponytail architecture). +""" + +import json +import sqlite3 +import asyncio +from typing import Dict, Any, List, Optional +from datetime import datetime + +# In-memory cancellation registry for instantaneous cooperative cancellation +CANCELLED_TASK_IDS = set() + + +def create_task( + conn: sqlite3.Connection, + user_id: int, + task_type: str, + params: Optional[Dict[str, Any]] = None, + total: int = 0, + message: str = "Задача поставлена в очередь", +) -> int: + """ + Creates a new background task in 'pending' status. + """ + params_json = json.dumps(params or {}, ensure_ascii=False) + cursor = conn.execute( + """ + INSERT INTO background_tasks ( + user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, + created_at, updated_at + ) VALUES (?, ?, 'pending', 0, ?, ?, ?, '{}', '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, + (user_id, task_type, total, message, params_json), + ) + return cursor.lastrowid + + +def update_task_progress( + conn: sqlite3.Connection, + task_id: int, + current: int, + total: Optional[int] = None, + message: Optional[str] = None, +): + """ + Updates the progress and status of a running task. + """ + if total is not None and message is not None: + conn.execute( + """ + UPDATE background_tasks + SET status = 'running', + progress_current = ?, + progress_total = ?, + message = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (current, total, message, task_id), + ) + elif total is not None: + conn.execute( + """ + UPDATE background_tasks + SET status = 'running', + progress_current = ?, + progress_total = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (current, total, task_id), + ) + elif message is not None: + conn.execute( + """ + UPDATE background_tasks + SET status = 'running', + progress_current = ?, + message = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (current, message, task_id), + ) + else: + conn.execute( + """ + UPDATE background_tasks + SET status = 'running', + progress_current = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (current, task_id), + ) + + +def complete_task( + conn: sqlite3.Connection, + task_id: int, + result: Optional[Dict[str, Any]] = None, + message: str = "Задача успешно завершена", +): + """ + Marks a background task as completed. + """ + result_json = json.dumps(result or {}, ensure_ascii=False) + conn.execute( + """ + UPDATE background_tasks + SET status = 'completed', + result_json = ?, + message = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (result_json, message, task_id), + ) + CANCELLED_TASK_IDS.discard(task_id) + + +def fail_task( + conn: sqlite3.Connection, + task_id: int, + error_message: str, +): + """ + Marks a background task as failed with an error message. + """ + conn.execute( + """ + UPDATE background_tasks + SET status = 'failed', + error_message = ?, + message = 'Ошибка выполнения задачи', + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status != 'cancelled' + """, + (error_message, task_id), + ) + CANCELLED_TASK_IDS.discard(task_id) + + +def cancel_task(conn: sqlite3.Connection, task_id: int, user_id: int) -> bool: + """ + Cooperatively cancels a task. + """ + CANCELLED_TASK_IDS.add(task_id) + cursor = conn.execute( + """ + UPDATE background_tasks + SET status = 'cancelled', + message = 'Задача отменена пользователем', + updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND user_id = ? AND status IN ('pending', 'running') + """, + (task_id, user_id), + ) + return cursor.rowcount > 0 + + +def is_task_cancelled(task_id: int, conn: Optional[sqlite3.Connection] = None) -> bool: + """ + Checks if a task was requested to be cancelled. + """ + if task_id in CANCELLED_TASK_IDS: + return True + if conn: + row = conn.execute( + "SELECT status FROM background_tasks WHERE id = ?", (task_id,) + ).fetchone() + if row and row[0] == "cancelled": + CANCELLED_TASK_IDS.add(task_id) + return True + return False + + +def get_task_status(conn: sqlite3.Connection, task_id: int, user_id: Optional[int] = None) -> Optional[Dict[str, Any]]: + """ + Retrieves full details of a specific task. + """ + if user_id is not None: + row = conn.execute( + """ + SELECT id, user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, created_at, updated_at + FROM background_tasks + WHERE id = ? AND user_id = ? + """, + (task_id, user_id), + ).fetchone() + else: + row = conn.execute( + """ + SELECT id, user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, created_at, updated_at + FROM background_tasks + WHERE id = ? + """, + (task_id,), + ).fetchone() + + if not row: + return None + + return _format_task_row(row) + + +def get_active_task(conn: sqlite3.Connection, user_id: int, task_type: Optional[str] = None) -> Optional[Dict[str, Any]]: + """ + Returns the most recent active (pending or running) task for user. + """ + if task_type: + row = conn.execute( + """ + SELECT id, user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, created_at, updated_at + FROM background_tasks + WHERE user_id = ? AND status IN ('pending', 'running') AND task_type = ? + ORDER BY id DESC LIMIT 1 + """, + (user_id, task_type), + ).fetchone() + else: + row = conn.execute( + """ + SELECT id, user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, created_at, updated_at + FROM background_tasks + WHERE user_id = ? AND status IN ('pending', 'running') + ORDER BY id DESC LIMIT 1 + """, + (user_id,), + ).fetchone() + + if not row: + return None + + return _format_task_row(row) + + +def list_user_tasks(conn: sqlite3.Connection, user_id: int, limit: int = 10) -> List[Dict[str, Any]]: + """ + Returns recent background tasks for user. + """ + rows = conn.execute( + """ + SELECT id, user_id, task_type, status, progress_current, progress_total, + message, params_json, result_json, error_message, created_at, updated_at + FROM background_tasks + WHERE user_id = ? + ORDER BY id DESC LIMIT ? + """, + (user_id, limit), + ).fetchall() + + return [_format_task_row(r) for r in rows] + + +def _format_task_row(row: Any) -> Dict[str, Any]: + """Helper to convert sqlite3.Row or tuple to a clean JSON-friendly dict.""" + params = {} + result = {} + try: + params = json.loads(row["params_json"] or "{}") + except Exception: + pass + try: + result = json.loads(row["result_json"] or "{}") + except Exception: + pass + + total = row["progress_total"] or 0 + current = row["progress_current"] or 0 + percent = round((current / total) * 100) if total > 0 else 0 + + return { + "id": row["id"], + "user_id": row["user_id"], + "task_type": row["task_type"], + "status": row["status"], + "progress_current": current, + "progress_total": total, + "progress_percent": percent, + "message": row["message"] or "", + "params": params, + "result": result, + "error_message": row["error_message"] or "", + "created_at": str(row["created_at"]), + "updated_at": str(row["updated_at"]), + } diff --git a/tests/test_api.py b/tests/test_api.py index 95bbd8b..3d89aa6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1393,11 +1393,10 @@ def test_cefr_testing_and_gap_ingestion(): assert "clue" in q1 assert len(q1["options"]) == 4 - # 5. Answer INCORRECTLY -> Automated Dual Gap Ingestion (target + selected option) # Pick a distractor from the question's actual options - # Find option that is not correct (we test with the first distractor) - # Let's inspect options and find one that is wrong - wrong_opt = [opt for opt in q1["options"] if opt.lower() != q1.get("term", "").lower()][0] + with get_db() as conn: + correct_term = conn.execute("SELECT term FROM cefr_questions WHERE id = ?", (q1["id"],)).fetchone()["term"] + wrong_opt = [opt for opt in q1["options"] if opt.lower() != correct_term.lower()][0] ans_wrong = client.post("/api/cefr/answer", json={ "question_id": q1["id"], @@ -1576,6 +1575,77 @@ def test_grammar_engine_and_handwritten_journaling(): assert u1_after["handwritten_logged"] is True +def test_background_tasks_and_book_mining(): + # 1. Unauthenticated check + client.cookies.clear() + unauth = client.post("/api/tasks/mine-books", json={"level": "A1", "count": 2}) + assert unauth.status_code == 401 + + # 2. Login + login_res = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"}) + assert login_res.status_code == 200 + + # 3. Create a book with known vocabulary + book_resp = client.post("/api/books", json={ + "title": "Adventures in London", + "author": "Arthur Conan Doyle", + "content": "Chapter 1: The Arrival\nThey decided to invite their neighbor for breakfast. The journey was long and dangerous.\n\nChapter 2: The Mystery\nHe could not remember where he left his expensive ticket.", + "source_type": "text" + }) + assert book_resp.status_code == 200 + book_id = book_resp.json()["id"] + + # 4. Start mining task + mine_resp = client.post("/api/tasks/mine-books", json={ + "book_id": book_id, + "level": "A1", + "count": 2 + }) + assert mine_resp.status_code == 200 + data = mine_resp.json() + assert data["status"] == "ok" + assert "task_id" in data + task_id = data["task_id"] + + # 5. Check task status via active and ID endpoint + active_resp = client.get("/api/tasks/active") + assert active_resp.status_code == 200 + + # Wait briefly or poll task status + import time + t_data = None + for _ in range(30): + t_res = client.get(f"/api/tasks/{task_id}") + assert t_res.status_code == 200 + t_data = t_res.json()["task"] + if t_data["status"] in ("completed", "failed", "cancelled"): + break + time.sleep(0.1) + + assert t_data is not None + assert t_data["status"] == "completed" + assert t_data["progress_current"] > 0 + + # 6. Verify quiz retrieval includes source book metadata + quiz_res = client.get("/api/cefr/quiz?level=A1&count=20") + assert quiz_res.status_code == 200 + items = quiz_res.json()["quiz"] + book_items = [q for q in items if q.get("source_book_title") == "Adventures in London"] + assert len(book_items) > 0 + assert book_items[0]["source_book_title"] == "Adventures in London" + assert "source_chapter_title" in book_items[0] + + # 7. Test task cancellation + from app.tasks import create_task + with get_db() as conn: + task_id_pending = create_task(conn, 1, "mine_books", total=10) + cancel_resp = client.post(f"/api/tasks/{task_id_pending}/cancel") + assert cancel_resp.status_code == 200 + assert cancel_resp.json()["status"] == "ok" + assert cancel_resp.json()["cancelled"] is True + + +