feat: background tasks manager and book sentence mining for CEFR tests

app/cefr.py

@@ -1968,7 +1968,8 @@ def get_cefr_quiz_batch(
     if mode == "mistakes":
         rows = conn.execute(
             """
-            SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json
+            SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json,
+                   q.source_book_id, q.source_book_title, q.source_chapter_title
             FROM cefr_questions q
             JOIN cefr_progress p ON q.id = p.question_id
             WHERE p.user_id = ? AND q.level = ? AND p.is_mastered = 0 AND p.mistakes > 0
@@ -1981,13 +1982,14 @@ def get_cefr_quiz_batch(
         # 1. Unattempted questions
         rows = conn.execute(
             """
-            SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json
+            SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json,
+                   q.source_book_id, q.source_book_title, q.source_chapter_title
             FROM cefr_questions q
             LEFT JOIN cefr_progress p ON q.id = p.question_id AND p.user_id = ?
             WHERE q.level = ? AND (p.question_id IS NULL OR p.is_mastered = 0)
             ORDER BY CASE WHEN p.question_id IS NULL THEN 0 ELSE 1 END,
                      p.last_tested_at ASC,
-                     q.id ASC
+                     q.id DESC
             LIMIT ?
             """,
             (user_id, clean_level, count),
@@ -2000,10 +2002,11 @@ def get_cefr_quiz_batch(
             placeholders = ",".join(["?"] * len(existing_ids)) if existing_ids else "-1"
             extra = conn.execute(
                 f"""
-                SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json
+                SELECT q.id, q.level, q.sentence, q.translation, q.pos, q.options_json,
+                       q.source_book_id, q.source_book_title, q.source_chapter_title
                 FROM cefr_questions q
                 WHERE q.level = ? AND q.id NOT IN ({placeholders})
-                ORDER BY q.id ASC
+                ORDER BY q.id DESC
                 LIMIT ?
                 """,
                 [clean_level] + existing_ids + [needed],
@@ -2020,14 +2023,18 @@ def get_cefr_quiz_batch(
         shuffled_options = list(options)
         random.shuffle(shuffled_options)
 
-        quiz_items.append({
+        item = {
             "id": r["id"],
             "level": r["level"],
             "sentence": r["sentence"],
             "clue": r["translation"],
             "pos": r["pos"] or "",
             "options": shuffled_options,
-        })
+        }
+        if "source_book_title" in r.keys() and r["source_book_title"]:
+            item["source_book_title"] = r["source_book_title"]
+            item["source_chapter_title"] = r["source_chapter_title"] or ""
+        quiz_items.append(item)
 
     return quiz_items
 
@@ -2212,6 +2219,8 @@ def process_cefr_answer(
         "selected_card_id": selected_card_info["card_id"] if selected_card_info else None,
         "selected_card_term": selected_clean if not is_correct else None,
         "selected_translation": selected_card_info["translation"] if selected_card_info else None,
+        "source_book_title": q_row["source_book_title"] if "source_book_title" in q_row.keys() else None,
+        "source_chapter_title": q_row["source_chapter_title"] if "source_chapter_title" in q_row.keys() else None,
         "options_breakdown": options_breakdown,
     }
 

app/database.py

@@ -216,12 +216,71 @@ def _migrate(conn: sqlite3.Connection):
         explanation TEXT DEFAULT '',
         frequency_rank INTEGER DEFAULT 0,
         created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-        UNIQUE(level, term COLLATE NOCASE)
+        source_book_id INTEGER DEFAULT NULL,
+        source_book_title TEXT DEFAULT '',
+        source_chapter_title TEXT DEFAULT ''
     );
     """)
     conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);")
     conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);")
 
+    # Safe migration: if old table had UNIQUE(level, term), migrate to allow rich book-mined sentences
+    cefr_ddl_row = conn.execute(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='cefr_questions'"
+    ).fetchone()
+    if cefr_ddl_row and "UNIQUE(level, term" in cefr_ddl_row[0]:
+        conn.execute("PRAGMA foreign_keys = OFF;")
+        conn.execute("""
+        CREATE TABLE cefr_questions_new (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            level TEXT NOT NULL,
+            term TEXT NOT NULL COLLATE NOCASE,
+            translation TEXT NOT NULL,
+            pos TEXT DEFAULT '',
+            phonetic TEXT DEFAULT '',
+            sentence TEXT NOT NULL,
+            options_json TEXT NOT NULL,
+            explanation TEXT DEFAULT '',
+            frequency_rank INTEGER DEFAULT 0,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            source_book_id INTEGER DEFAULT NULL,
+            source_book_title TEXT DEFAULT '',
+            source_chapter_title TEXT DEFAULT ''
+        );
+        """)
+        conn.execute("""
+        INSERT INTO cefr_questions_new (
+            id, level, term, translation, pos, phonetic, sentence, options_json,
+            explanation, frequency_rank, created_at, source_book_id, source_book_title, source_chapter_title
+        )
+        SELECT id, level, term, translation, pos, phonetic, sentence, options_json,
+               explanation, frequency_rank, created_at,
+               COALESCE(source_book_id, NULL),
+               COALESCE(source_book_title, ''),
+               COALESCE(source_chapter_title, '')
+        FROM cefr_questions;
+        """)
+        conn.execute("DROP TABLE cefr_questions;")
+        conn.execute("ALTER TABLE cefr_questions_new RENAME TO cefr_questions;")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_level ON cefr_questions(level);")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_cefr_q_term ON cefr_questions(term COLLATE NOCASE);")
+        conn.execute("PRAGMA foreign_keys = ON;")
+
+    # Safe migration: add book source metadata columns if not present
+    existing_cefr_cols = {
+        row[1] for row in conn.execute("PRAGMA table_info(cefr_questions)").fetchall()
+    }
+    cefr_migrations = [
+        ("source_book_id", "INTEGER DEFAULT NULL"),
+        ("source_book_title", "TEXT DEFAULT ''"),
+        ("source_chapter_title", "TEXT DEFAULT ''"),
+    ]
+    for col, definition in cefr_migrations:
+        if col not in existing_cefr_cols:
+            conn.execute(f"ALTER TABLE cefr_questions ADD COLUMN {col} {definition}")
+
+    conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_cefr_q_level_sentence ON cefr_questions(level, sentence);")
+
     conn.execute("""
     CREATE TABLE IF NOT EXISTS cefr_progress (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -299,6 +358,27 @@ def _migrate(conn: sqlite3.Connection):
     except Exception:
         pass
 
+    # Ensure background_tasks table exists (Step 1: Background Tasks Engine)
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS background_tasks (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        user_id INTEGER NOT NULL,
+        task_type TEXT NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        progress_current INTEGER DEFAULT 0,
+        progress_total INTEGER DEFAULT 0,
+        message TEXT DEFAULT '',
+        params_json TEXT DEFAULT '{}',
+        result_json TEXT DEFAULT '{}',
+        error_message TEXT DEFAULT '',
+        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    );
+    """)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_user_status ON background_tasks(user_id, status);")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_bg_tasks_status ON background_tasks(status);")
+
 
 
 def _seed_offline_dictionary(conn: sqlite3.Connection):

app/main.py

@@ -12,6 +12,8 @@ from fastapi.middleware.gzip import GZipMiddleware
 from pydantic import BaseModel, Field
 import httpx
 
+import asyncio
+
 from app import config
 from app.database import init_db, get_db
 from app.auth import (
@@ -45,6 +47,14 @@ from app.grammar import (
     record_grammar_completion,
     update_handwritten_status,
 )
+from app.tasks import (
+    create_task,
+    get_task_status,
+    get_active_task,
+    list_user_tasks,
+    cancel_task,
+)
+from app.miner import run_mining_task_background
 
 
 
@@ -210,6 +220,12 @@ class GrammarHandwrittenRequest(BaseModel):
     handwritten: bool
 
 
+class MineBooksRequest(BaseModel):
+    book_id: Optional[int] = None
+    level: str = "A1"
+    count: int = Field(15, ge=1, le=50)
+
+
 
 
 
@@ -1557,3 +1573,103 @@ async def api_grammar_handwritten(
         return update_handwritten_status(conn, user["id"], req.unit_id, req.handwritten)
 
 
+# ---------------------------------------------------------------------------
+# Background Tasks & Book Sentence Mining Endpoints
+# ---------------------------------------------------------------------------
+
+@app.post("/api/tasks/mine-books")
+async def api_tasks_mine_books(
+    req: MineBooksRequest,
+    user: dict = Depends(require_user),
+):
+    """
+    Starts an asynchronous background task to mine CEFR Cloze Tap questions
+    from user's uploaded books (EPUB, TXT, PDF).
+    """
+    level = req.level.upper().strip()
+    if level not in ["A1", "A2", "B1", "B2", "ALL"]:
+        raise HTTPException(status_code=400, detail="Invalid CEFR level")
+
+    count = max(1, min(req.count, 50))
+
+    with get_db() as conn:
+        active = get_active_task(conn, user["id"], task_type="mine_books")
+        if active:
+            return {
+                "status": "already_running",
+                "message": "Сборка уже выполняется в фоне",
+                "task": active,
+            }
+
+        task_id = create_task(
+            conn,
+            user_id=user["id"],
+            task_type="mine_books",
+            params={
+                "book_id": req.book_id,
+                "level": level,
+                "count": count,
+            },
+            total=count,
+        )
+
+    # Launch background asyncio task
+    asyncio.create_task(
+        run_mining_task_background(
+            task_id=task_id,
+            user_id=user["id"],
+            book_id=req.book_id,
+            target_level=level,
+            target_count=count,
+        )
+    )
+
+    return {
+        "status": "ok",
+        "task_id": task_id,
+        "message": f"Запущена фоновая сборка тестов ({level})",
+    }
+
+
+@app.get("/api/tasks/active")
+async def api_tasks_active(
+    task_type: Optional[str] = Query(None),
+    user: dict = Depends(require_user),
+):
+    """
+    Returns current user's active background task (if any).
+    """
+    with get_db() as conn:
+        task = get_active_task(conn, user["id"], task_type=task_type)
+        return {"task": task}
+
+
+@app.get("/api/tasks/{task_id}")
+async def api_tasks_status(
+    task_id: int,
+    user: dict = Depends(require_user),
+):
+    """
+    Returns status and progress of a background task.
+    """
+    with get_db() as conn:
+        task = get_task_status(conn, task_id, user_id=user["id"])
+        if not task:
+            raise HTTPException(status_code=404, detail="Task not found")
+        return {"task": task}
+
+
+@app.post("/api/tasks/{task_id}/cancel")
+async def api_tasks_cancel(
+    task_id: int,
+    user: dict = Depends(require_user),
+):
+    """
+    Cancels a running background task.
+    """
+    with get_db() as conn:
+        cancelled = cancel_task(conn, task_id, user_id=user["id"])
+        return {"status": "ok" if cancelled else "not_found", "cancelled": cancelled}
+
+
+

app/miner.py

@@ -0,0 +1,462 @@
+"""
+Book Sentence Mining & Automated CEFR Question Generator.
+Extracts real, natural sentences from user's uploaded books (EPUB, PDF, TXT),
+identifies target CEFR vocabulary, blanks out the target word,
+and matches plausible distractors of the same Part of Speech.
+100% offline, zero-dependency, pure Python stdlib + SQLite.
+"""
+
+import re
+import json
+import random
+import sqlite3
+import asyncio
+from typing import Dict, Any, List, Optional, Tuple
+
+from app.tasks import update_task_progress, complete_task, fail_task, is_task_cancelled
+
+# Curated Oxford 3000/5000 benchmark lexicon categorized by level and part-of-speech
+CEFR_LEXICON: Dict[str, Dict[str, List[str]]] = {
+    "A1": {
+        "adjective": [
+            "expensive", "cheap", "careful", "dangerous", "healthy", "boring", "famous",
+            "simple", "wonderful", "wrong", "clean", "heavy", "narrow", "empty",
+            "hungry", "thirsty", "quiet", "loud", "clever"
+        ],
+        "verb": [
+            "remember", "decide", "invite", "borrow", "listen", "repeat", "travel",
+            "welcome", "laugh", "shout", "wash", "catch", "throw", "build", "climb"
+        ],
+        "noun": [
+            "weather", "breakfast", "village", "journey", "holiday", "hospital", "library",
+            "message", "package", "question", "station", "ticket", "uniform", "bottle",
+            "island", "pocket", "mirror", "market", "bridge", "corner", "cloud", "floor",
+            "blanket"
+        ],
+    },
+    "A2": {
+        "adjective": [
+            "necessary", "strange", "patient", "proud", "separate", "urgent", "wealthy",
+            "afraid", "complete", "connected", "disappointed", "jealous", "slow"
+        ],
+        "verb": [
+            "discover", "improve", "protect", "remain", "suggest", "compete", "earn",
+            "refuse", "repair", "warn", "attend", "celebrate", "afford", "arrange",
+            "avoid", "compare", "contain", "damage", "depend", "describe", "discuss",
+            "doubt", "expect", "explain", "fail", "guess", "manage", "notice", "succeed"
+        ],
+        "noun": [
+            "accident", "advice", "customer", "equipment", "salary", "traffic", "transport",
+            "benefit", "habit", "choice", "condition", "crowd", "culture", "degree",
+            "effect", "energy", "environment", "experience", "flight", "future", "nature",
+            "opinion", "reason"
+        ],
+    },
+    "B1": {
+        "adjective": [
+            "mysterious", "ambitious", "curious", "efficient", "fluent", "frequent",
+            "genuine", "persistent", "reasonable", "sensible", "suitable", "temporary",
+            "universal", "valuable", "vulnerable", "confident", "essential", "critical",
+            "adequate", "apparent", "accurate"
+        ],
+        "verb": [
+            "accomplish", "convince", "emphasize", "intend", "obtain", "predict",
+            "threaten", "approve", "broadcast", "cooperate", "distinguish", "eliminate",
+            "guarantee", "identify", "achieve", "consider", "develop", "encourage",
+            "hesitate", "influence", "interrupt", "maintain", "negotiate", "persuade"
+        ],
+        "noun": [
+            "secret", "anxiety", "circumstance", "community", "decision", "mystery",
+            "opportunity", "relationship", "requirement", "consequence", "behavior",
+            "challenge"
+        ],
+    },
+    "B2": {
+        "adjective": [
+            "meticulous", "novel", "relentless", "subtle", "tangible", "ubiquitous",
+            "unprecedented", "viable", "zealous", "ambiguous", "benevolent", "complacent",
+            "diligent", "ephemeral", "formidable", "innovative", "gregarious", "homogeneous",
+            "lucrative", "eligible", "arbitrary", "coherent", "comprehensive", "crucial",
+            "feasible", "fundamental", "inevitable", "inherent", "plausible", "preliminary",
+            "profound", "resilient", "simultaneous", "sophisticated", "substantial"
+        ],
+        "verb": [
+            "deteriorate", "scrutinize", "withstand", "yield", "diminish", "undermine",
+            "fluctuate", "facilitate", "integrate", "deduce", "accommodate", "advocate",
+            "allocate", "convey", "elaborate", "enhance", "justify", "perceive"
+        ],
+        "noun": [
+            "discrepancy", "compromise", "consensus", "ambiguity", "contradiction",
+            "paradox", "hypothesis"
+        ],
+    },
+}
+
+# Reverse lookup index: term -> list of (level, pos)
+_WORD_INDEX: Dict[str, List[Tuple[str, str]]] = {}
+for _lvl, _pos_map in CEFR_LEXICON.items():
+    for _pos, _words in _pos_map.items():
+        for _w in _words:
+            _w_low = _w.lower()
+            if _w_low not in _WORD_INDEX:
+                _WORD_INDEX[_w_low] = []
+            _WORD_INDEX[_w_low].append((_lvl, _pos))
+
+# Common abbreviations to prevent false splits
+_HONORIFICS = ["Mr.", "Mrs.", "Ms.", "Dr.", "Prof.", "St.", "vs.", "etc.", "e.g.", "i.e.", "Jr.", "Sr."]
+
+
+def split_into_clean_sentences(text: str) -> List[str]:
+    """
+    Safely splits narrative prose into clean, grammatically complete sentences.
+    Protects honorifics and abbreviations from being split incorrectly.
+    """
+    if not text:
+        return []
+
+    # Protect common abbreviations and titles
+    protected = text
+    for abbr in _HONORIFICS:
+        protected = protected.replace(abbr, abbr.replace(".", "§DOT§"))
+
+    # Split on sentence-ending punctuation (.!?) followed by space or newline
+    raw_splits = re.split(r"(?<=[.!?])\s+", protected)
+
+    clean_sentences = []
+    for s in raw_splits:
+        restored = s.replace("§DOT§", ".").strip()
+        # Flatten multiple spaces and line breaks
+        flattened = re.sub(r"\s+", " ", restored)
+
+        # Basic validity filters:
+        # 1. Skip dialogue lines starting with quotation marks or dashes
+        if flattened.startswith('"') or flattened.startswith('“') or flattened.startswith('—'):
+            continue
+        # 2. Skip all-caps headers
+        if flattened.isupper():
+            continue
+        # 3. Must end with standard punctuation
+        if not re.search(r"[.!?]$", flattened):
+            continue
+
+        words = re.findall(r"\b[A-Za-z]+\b", flattened)
+        # Optimal pedagogical length: 8 to 24 words
+        if 8 <= len(words) <= 24:
+            clean_sentences.append(flattened)
+
+    return clean_sentences
+
+
+def find_distractors(
+    conn: sqlite3.Connection,
+    target_term: str,
+    target_pos: str,
+    target_level: str,
+    count: int = 3,
+) -> List[str]:
+    """
+    Finds 3 clean, plausible distractors of the same CEFR level and part of speech.
+    First prioritizes words from the curated CEFR lexicon, then falls back to offline_dictionary.
+    """
+    clean_target = target_term.lower().strip()
+    distractors = []
+
+    # 1. Primary pool: words from the exact same level and POS in our curated lexicon
+    level_dict = CEFR_LEXICON.get(target_level, {})
+    pool = [w.lower() for w in level_dict.get(target_pos, []) if w.lower() != clean_target]
+
+    if len(pool) >= count:
+        random.shuffle(pool)
+        distractors = pool[:count]
+    else:
+        # Expand pool with adjacent levels if needed
+        adjacent_levels = ["A2", "B1", "B2", "A1"]
+        for adj_lvl in adjacent_levels:
+            adj_pool = [w.lower() for w in CEFR_LEXICON.get(adj_lvl, {}).get(target_pos, []) if w.lower() != clean_target and w.lower() not in pool]
+            pool.extend(adj_pool)
+            if len(pool) >= count:
+                break
+        random.shuffle(pool)
+        distractors = pool[:count]
+
+    # Ensure we have required count, else query offline_dictionary
+    if len(distractors) < count:
+        pos_prefix = target_pos.capitalize()
+        extra_rows = conn.execute(
+            """
+            SELECT term FROM offline_dictionary
+            WHERE pos LIKE ? AND length(term) BETWEEN 4 AND 12
+              AND term NOT LIKE '% %' AND term NOT LIKE '%-%'
+              AND lower(term) != ?
+            ORDER BY RANDOM() LIMIT 20
+            """,
+            (f"{pos_prefix}%", clean_target),
+        ).fetchall()
+        for r in extra_rows:
+            w = r["term"].lower()
+            if w != clean_target and w not in distractors:
+                distractors.append(w)
+                if len(distractors) >= count:
+                    break
+
+    return distractors[:count]
+
+
+def clean_translation(trans: str) -> str:
+    """Cleans up raw translation strings, removes slash phonetic residues and romanizations."""
+    if not trans:
+        return ""
+    t = re.sub(r"/[^/]+/?", "", trans)
+    parts = [p.strip() for p in re.split(r"[,;]\s*", t) if p.strip()]
+    clean_parts = []
+    for p in parts:
+        if re.search(r"[а-яА-ЯёЁ]", p):
+            clean_p = re.sub(r"[a-zA-ZáéíóúÁÉÍÓÚ]", "", p).strip()
+            if clean_p:
+                clean_parts.append(clean_p)
+    return ", ".join(clean_parts[:3]) if clean_parts else trans.split(",")[0].strip()
+
+
+async def mine_book_questions(
+    conn: sqlite3.Connection,
+    user_id: int,
+    book_id: Optional[int],
+    target_level: str,
+    target_count: int = 15,
+    task_id: Optional[int] = None,
+) -> Dict[str, Any]:
+    """
+    Asynchronously mines candidate sentences from user's book library,
+    identifies CEFR vocabulary targets, generates balanced multiple-choice questions,
+    and stores them directly into cefr_questions.
+    """
+    clean_target_level = target_level.upper().strip() if target_level else "ALL"
+
+    # Fetch candidate books
+    if book_id and book_id > 0:
+        book_rows = conn.execute(
+            "SELECT id, title, chapters_json FROM books WHERE id = ? AND user_id = ?",
+            (book_id, user_id),
+        ).fetchall()
+    else:
+        book_rows = conn.execute(
+            "SELECT id, title, chapters_json FROM books WHERE user_id = ? ORDER BY updated_at DESC",
+            (user_id,),
+        ).fetchall()
+
+    if not book_rows:
+        if task_id:
+            fail_task(conn, task_id, "В вашей библиотеке не найдено книг для анализа.")
+        return {"status": "error", "message": "No books found"}
+
+    mined_questions: List[Dict[str, Any]] = []
+    mined_count = 0
+    scanned_sentences = 0
+
+    # Collect existing sentences already tested in cefr_questions to prevent duplicate questions
+    existing_sentences = set(
+        r[0] for r in conn.execute("SELECT sentence FROM cefr_questions").fetchall()
+    )
+    # Track terms mined in THIS session to ensure diversity of vocabulary
+    session_terms = set()
+
+    for book in book_rows:
+        b_id = book["id"]
+        b_title = book["title"]
+        try:
+            chapters = json.loads(book["chapters_json"] or "[]")
+        except Exception:
+            chapters = []
+
+        for ch_idx, ch in enumerate(chapters):
+            if is_task_cancelled(task_id, conn):
+                break
+
+            ch_title = ch.get("title") or f"Глава {ch_idx + 1}"
+            ch_content = ch.get("content", "")
+            sentences = split_into_clean_sentences(ch_content)
+
+            for sentence in sentences:
+                if is_task_cancelled(task_id, conn):
+                    break
+
+                scanned_sentences += 1
+                words = re.findall(r"\b[A-Za-z]+\b", sentence)
+
+                for word in words:
+                    w_low = word.lower()
+                    if w_low in session_terms:
+                        continue
+
+                    # Check if word is in our target CEFR lexicon
+                    matches = _WORD_INDEX.get(w_low, [])
+                    if not matches:
+                        continue
+
+                    # Filter by target level if requested
+                    selected_match = None
+                    if clean_target_level != "ALL":
+                        for lvl, pos in matches:
+                            if lvl == clean_target_level:
+                                selected_match = (lvl, pos)
+                                break
+                    else:
+                        selected_match = matches[0]
+
+                    if not selected_match:
+                        continue
+
+                    lvl, pos = selected_match
+
+                    # Create cloze gap (only first match)
+                    blanked_sentence = re.sub(
+                        r"\b" + re.escape(word) + r"\b",
+                        "___",
+                        sentence,
+                        count=1,
+                    )
+                    if blanked_sentence in existing_sentences:
+                        continue
+
+                    # Look up translation and phonetic from offline_dictionary
+                    dict_row = conn.execute(
+                        "SELECT translation, phonetic, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE",
+                        (w_low,),
+                    ).fetchone()
+                    if not dict_row or not dict_row["translation"]:
+                        continue
+
+                    translation = clean_translation(dict_row["translation"])
+                    phonetic = dict_row["phonetic"] or ""
+
+                    # Find 3 suitable distractors
+                    distractors = find_distractors(conn, w_low, pos, lvl, count=3)
+                    if len(distractors) < 3:
+                        continue
+
+                    options = [w_low] + distractors
+                    random.shuffle(options)
+
+                    explanation = f"{word.capitalize()} ({translation}) — пример из книги «{b_title}» ({ch_title})."
+
+                    # Insert into cefr_questions
+                    try:
+                        cursor = conn.execute(
+                            """
+                            INSERT OR IGNORE INTO cefr_questions (
+                                level, term, translation, pos, phonetic,
+                                sentence, options_json, explanation,
+                                source_book_id, source_book_title, source_chapter_title
+                            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                            """,
+                            (
+                                lvl,
+                                w_low,
+                                translation,
+                                pos,
+                                phonetic,
+                                blanked_sentence,
+                                json.dumps(options, ensure_ascii=False),
+                                explanation,
+                                b_id,
+                                b_title,
+                                ch_title,
+                            ),
+                        )
+                        if cursor.rowcount > 0:
+                            existing_sentences.add(blanked_sentence)
+                            session_terms.add(w_low)
+                            mined_count += 1
+                            mined_questions.append({
+                                "id": cursor.lastrowid,
+                                "level": lvl,
+                                "term": w_low,
+                                "sentence": blanked_sentence,
+                                "book_title": b_title,
+                                "chapter_title": ch_title,
+                            })
+
+                            if task_id:
+                                update_task_progress(
+                                    conn,
+                                    task_id,
+                                    mined_count,
+                                    target_count,
+                                    f"Собрано {mined_count}/{target_count} вопросов из «{b_title}»",
+                                )
+                                conn.commit()
+
+                            # Cooperative yield
+                            await asyncio.sleep(0)
+
+                            if mined_count >= target_count:
+                                break
+                    except Exception:
+                        pass
+
+                if mined_count >= target_count:
+                    break
+            if mined_count >= target_count or is_task_cancelled(task_id, conn):
+                break
+        if mined_count >= target_count or is_task_cancelled(task_id, conn):
+            break
+
+    # Finalize task status
+    if task_id:
+        if is_task_cancelled(task_id, conn):
+            pass
+        elif mined_count > 0:
+            complete_task(
+                conn,
+                task_id,
+                result={
+                    "mined_count": mined_count,
+                    "target_count": target_count,
+                    "target_level": clean_target_level,
+                    "scanned_sentences": scanned_sentences,
+                },
+                message=f"Успешно создано {mined_count} новых вопросов из ваших книг!",
+            )
+            conn.commit()
+        else:
+            complete_task(
+                conn,
+                task_id,
+                result={"mined_count": 0, "scanned_sentences": scanned_sentences},
+                message="Все найденные слова книги уже есть в вашей базе вопросов.",
+            )
+            conn.commit()
+
+    return {
+        "status": "ok",
+        "mined_count": mined_count,
+        "target_count": target_count,
+        "questions": mined_questions,
+    }
+
+
+async def run_mining_task_background(
+    task_id: int,
+    user_id: int,
+    book_id: Optional[int],
+    target_level: str,
+    target_count: int = 15,
+) -> None:
+    """Entry point for asyncio background task execution with isolated DB connection."""
+    from app.database import get_db_connection
+    conn = get_db_connection()
+    try:
+        await mine_book_questions(
+            conn=conn,
+            user_id=user_id,
+            book_id=book_id,
+            target_level=target_level,
+            target_count=target_count,
+            task_id=task_id,
+        )
+    except Exception as e:
+        fail_task(conn, task_id, f"Ошибка сборки: {str(e)}")
+        conn.commit()
+    finally:
+        conn.close()
+

app/static/css/style.css

@@ -4276,7 +4276,89 @@ body {
 }
 
 .cefr-header {
-    margin-bottom: 28px;
+    display: flex;
+    justify-content: space-between;
+    align-items: flex-start;
+    gap: 16px;
+    margin-bottom: 24px;
+    flex-wrap: wrap;
+}
+
+.cefr-header-actions {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+}
+
+/* Background Task Banner in CEFR Overview */
+.cefr-task-banner {
+    background: linear-gradient(135deg, rgba(59, 130, 246, 0.12), rgba(16, 185, 129, 0.08));
+    border: 1px solid rgba(59, 130, 246, 0.3);
+    border-radius: var(--radius-md);
+    padding: 14px 18px;
+    margin-bottom: 24px;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.08);
+}
+
+.task-banner-inner {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    gap: 16px;
+}
+
+.task-banner-content {
+    flex: 1;
+}
+
+.task-banner-title {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    font-size: 14px;
+    color: var(--text-main);
+    margin-bottom: 4px;
+}
+
+.task-banner-counts {
+    font-size: 12px;
+    color: var(--text-muted);
+    font-weight: 500;
+    margin-left: auto;
+}
+
+.task-banner-msg {
+    font-size: 13px;
+    color: var(--text-muted);
+}
+
+/* Book Source Badge in Question Card & Feedback */
+.cefr-book-badge {
+    display: inline-flex;
+    align-items: center;
+    gap: 4px;
+    font-size: 12px;
+    font-weight: 600;
+    color: #3b82f6;
+    background: rgba(59, 130, 246, 0.1);
+    border: 1px solid rgba(59, 130, 246, 0.25);
+    padding: 2px 10px;
+    border-radius: var(--radius-sm);
+    max-width: 280px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+
+.fb-source-citation {
+    font-size: 12.5px;
+    font-weight: 500;
+    color: #3b82f6;
+    background: rgba(59, 130, 246, 0.08);
+    border-left: 3px solid #3b82f6;
+    padding: 6px 10px;
+    border-radius: 0 var(--radius-sm) var(--radius-sm) 0;
+    margin: 8px 0;
 }
 
 .cefr-title-wrap h2 {

app/static/index.html

@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
     <title>Notes — Персональный тренажер и словарь английского языка</title>
     <meta name="description" content="Защищённый персональный тренажёр и словарь английского языка с интервальными повторениями SRS SM-2, контекстным поиском и озвучкой">
-    <link rel="stylesheet" href="/static/css/style.css?v=5.4">
+    <link rel="stylesheet" href="/static/css/style.css?v=5.5">
     
     <!-- PWA meta and icons -->
     <link rel="manifest" href="/manifest.json">
@@ -608,7 +608,15 @@
                         <h2>🎯 Уровневые тесты CEFR (A1 – B2)</h2>
                         <p class="subtitle">Эталонные тесты по международной шкале Oxford 3000/5000. Ошибки автоматически добавляются в ваш личный тренажёр SRS со статусом «В процессе»!</p>
                     </div>
+                    <div class="cefr-header-actions">
+                        <button type="button" id="cefr-mine-books-btn" class="btn btn-secondary">
+                            ⚡ Собрать тесты из книг
+                        </button>
                     </div>
+                </div>
+
+                <!-- Active Background Task Banner -->
+                <div id="cefr-task-banner" class="cefr-task-banner hidden"></div>
 
                 <div id="cefr-levels-grid" class="cefr-levels-grid">
                     <!-- Dynamic level cards rendered by cefr.js -->
@@ -1226,6 +1234,52 @@
         </div>
     </div>
 
+    <!-- Mine CEFR Questions from Books Modal -->
+    <div id="mine-books-modal" class="modal-backdrop hidden">
+        <div class="modal-window">
+            <div class="modal-header">
+                <h3>⚡ Сборка тестов из библиотеки книг</h3>
+                <button type="button" id="close-mine-books-modal-btn" class="icon-btn">✕</button>
+            </div>
+            <form id="mine-books-form">
+                <div class="modal-body">
+                    <p class="modal-desc" style="margin-bottom: 1rem; color: var(--text-muted); font-size: 0.9rem;">
+                        Сканирует ваши книги, находит аутентичные предложения с лексикой CEFR и создает тесты с выбором вариантов. Фоновая задача работает без задержек.
+                    </p>
+                    <div class="form-group">
+                        <label for="mine-books-select">Книга для анализа:</label>
+                        <select id="mine-books-select" class="input-styled">
+                            <option value="">📚 Все книги библиотеки</option>
+                        </select>
+                    </div>
+                    <div class="form-group">
+                        <label for="mine-level-select">Целевой уровень CEFR:</label>
+                        <select id="mine-level-select" class="input-styled">
+                            <option value="ALL">Все уровни (A1 – B2)</option>
+                            <option value="A1">A1 – Начальный</option>
+                            <option value="A2">A2 – Элементарный</option>
+                            <option value="B1">B1 – Средний</option>
+                            <option value="B2">B2 – Выше среднего</option>
+                        </select>
+                    </div>
+                    <div class="form-group">
+                        <label for="mine-count-input">Количество вопросов для сборки:</label>
+                        <select id="mine-count-input" class="input-styled">
+                            <option value="15" selected>15 вопросов</option>
+                            <option value="25">25 вопросов</option>
+                            <option value="35">35 вопросов</option>
+                            <option value="50">50 вопросов</option>
+                        </select>
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="button" id="cancel-mine-books-btn" class="btn btn-outline">Отмена</button>
+                    <button type="submit" id="submit-mine-books-btn" class="btn btn-primary">⚡ Запустить сборку</button>
+                </div>
+            </form>
+        </div>
+    </div>
+
     <!-- Reader Appearance Settings Popover / Bottom Sheet -->
     <div id="reader-settings-panel" class="reader-settings-panel hidden">
         <div class="sheet-drag-handle mobile-only"></div>
@@ -1383,7 +1437,7 @@
     <!-- Audio Element for native streaming -->
     <audio id="global-audio-player" preload="none"></audio>
 
-    <script type="module" src="/static/js/app.js?v=5.4"></script>
+    <script type="module" src="/static/js/app.js?v=5.5"></script>
 </body>
 </html>
 

app/static/js/modules/api.js

@@ -316,5 +316,41 @@ export const api = {
             if (resp.ok) return await resp.json();
             throw new Error(`Failed to update handwritten status: ${resp.status}`);
         }
+    },
+
+    // Background Tasks & Sentence Mining
+    tasks: {
+        async mineBooks(bookId = null, level = 'A1', count = 15) {
+            const resp = await fetch('/api/tasks/mine-books', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    book_id: bookId,
+                    level: level,
+                    count: count
+                })
+            });
+            if (resp.ok) return await resp.json();
+            throw new Error(`Failed to start mining task: ${resp.status}`);
+        },
+        async getActive(taskType = null) {
+            let url = '/api/tasks/active';
+            if (taskType) url += `?task_type=${encodeURIComponent(taskType)}`;
+            const resp = await fetch(url);
+            if (resp.ok) return await resp.json();
+            throw new Error(`Failed to get active task: ${resp.status}`);
+        },
+        async getStatus(taskId) {
+            const resp = await fetch(`/api/tasks/${taskId}`);
+            if (resp.ok) return await resp.json();
+            throw new Error(`Failed to get task status: ${resp.status}`);
+        },
+        async cancel(taskId) {
+            const resp = await fetch(`/api/tasks/${taskId}/cancel`, {
+                method: 'POST'
+            });
+            if (resp.ok) return await resp.json();
+            throw new Error(`Failed to cancel task: ${resp.status}`);
+        }
     }
 };

app/static/js/modules/cefr.js

@@ -64,6 +64,59 @@ export function initCefr() {
         });
     }
 
+    // Mine books modal setup
+    const mineBtn = document.getElementById('cefr-mine-books-btn');
+    const mineModal = document.getElementById('mine-books-modal');
+    const closeMineBtn = document.getElementById('close-mine-books-modal-btn');
+    const cancelMineBtn = document.getElementById('cancel-mine-books-btn');
+    const mineForm = document.getElementById('mine-books-form');
+    const bookSelect = document.getElementById('mine-books-select');
+
+    if (mineBtn && mineModal) {
+        mineBtn.addEventListener('click', async () => {
+            if (bookSelect) {
+                try {
+                    const books = await api.books.getAll();
+                    let optionsHtml = '<option value="">📚 Все книги библиотеки</option>';
+                    books.forEach(b => {
+                        optionsHtml += `<option value="${b.id}">📖 ${escapeHtml(b.title)}${b.author ? ` (${escapeHtml(b.author)})` : ''}</option>`;
+                    });
+                    bookSelect.innerHTML = optionsHtml;
+                } catch (e) {
+                    console.warn('Failed to load books for select:', e);
+                }
+            }
+            mineModal.classList.remove('hidden');
+        });
+
+        const closeMineModal = () => mineModal.classList.add('hidden');
+        if (closeMineBtn) closeMineBtn.addEventListener('click', closeMineModal);
+        if (cancelMineBtn) cancelMineBtn.addEventListener('click', closeMineModal);
+
+        if (mineForm) {
+            mineForm.addEventListener('submit', async (e) => {
+                e.preventDefault();
+                const bookIdVal = document.getElementById('mine-books-select')?.value;
+                const bookId = bookIdVal ? parseInt(bookIdVal, 10) : null;
+                const level = document.getElementById('mine-level-select')?.value || 'ALL';
+                const count = parseInt(document.getElementById('mine-count-input')?.value || '15', 10);
+
+                closeMineModal();
+                try {
+                    const res = await api.tasks.mineBooks(bookId, level, count);
+                    showToast(res.message || 'Сборка тестов запущена в фоне', 'info');
+                    if (res.task_id) {
+                        startTaskPolling(res.task_id);
+                    } else {
+                        checkActiveMiningTask();
+                    }
+                } catch (err) {
+                    showToast(`Ошибка запуска: ${err.message}`, 'error');
+                }
+            });
+        }
+    }
+
     // Keyboard navigation (1-4 for options, Space/Enter for next)
     window.addEventListener('keydown', (e) => {
         const cefrView = document.getElementById('cefr-view');
@@ -90,6 +143,106 @@ export function initCefr() {
     });
 }
 
+// -----------------------------------------------------------------------
+// Background Tasks Polling & Banner Controls
+// -----------------------------------------------------------------------
+let activeTaskPollTimer = null;
+
+export async function checkActiveMiningTask() {
+    try {
+        const res = await api.tasks.getActive('mine_books');
+        const task = res.task;
+        if (task && ['pending', 'running'].includes(task.status)) {
+            renderTaskBanner(task);
+            startTaskPolling(task.id);
+        } else {
+            hideTaskBanner();
+        }
+    } catch (err) {
+        console.warn('Failed to check active task:', err);
+    }
+}
+
+function startTaskPolling(taskId) {
+    if (activeTaskPollTimer) clearInterval(activeTaskPollTimer);
+    activeTaskPollTimer = setInterval(async () => {
+        try {
+            const res = await api.tasks.getStatus(taskId);
+            const task = res.task;
+            if (!task) {
+                clearInterval(activeTaskPollTimer);
+                activeTaskPollTimer = null;
+                hideTaskBanner();
+                return;
+            }
+
+            if (task.status === 'completed') {
+                clearInterval(activeTaskPollTimer);
+                activeTaskPollTimer = null;
+                hideTaskBanner();
+                showToast(task.message || 'Сборка тестов завершена!', 'success');
+                loadCefrOverview();
+            } else if (task.status === 'cancelled') {
+                clearInterval(activeTaskPollTimer);
+                activeTaskPollTimer = null;
+                hideTaskBanner();
+                showToast('Сборка тестов отменена', 'info');
+            } else if (task.status === 'failed') {
+                clearInterval(activeTaskPollTimer);
+                activeTaskPollTimer = null;
+                hideTaskBanner();
+                showToast(`Ошибка сборки: ${task.error_message || task.message}`, 'error');
+            } else {
+                renderTaskBanner(task);
+            }
+        } catch (err) {
+            console.warn('Poll error:', err);
+        }
+    }, 1500);
+}
+
+function renderTaskBanner(task) {
+    const banner = document.getElementById('cefr-task-banner');
+    if (!banner) return;
+    const pct = task.progress_percent || 0;
+    banner.innerHTML = `
+        <div class="task-banner-inner">
+            <div class="task-banner-content">
+                <div class="task-banner-title">
+                    <span class="spinner-sm"></span>
+                    <strong>⚡ Фоновая сборка тестов из книг</strong>
+                    <span class="task-banner-counts">${task.progress_current} / ${task.progress_total} (${pct}%)</span>
+                </div>
+                <div class="task-banner-msg">${escapeHtml(task.message || 'Анализ предложений...')}</div>
+                <div class="progress-bar-wrap" style="height: 6px; margin-top: 6px;">
+                    <div class="progress-bar-fill level-badge-b1" style="width: ${pct}%"></div>
+                </div>
+            </div>
+            <div class="task-banner-action">
+                <button type="button" class="btn btn-xs btn-outline task-cancel-btn" data-task-id="${task.id}">Отмена</button>
+            </div>
+        </div>
+    `;
+    banner.classList.remove('hidden');
+
+    banner.querySelector('.task-cancel-btn')?.addEventListener('click', async () => {
+        try {
+            await api.tasks.cancel(task.id);
+            showToast('Запрос на отмену отправлен...', 'info');
+        } catch (e) {
+            showToast('Не удалось отменить задачу', 'error');
+        }
+    });
+}
+
+function hideTaskBanner() {
+    const banner = document.getElementById('cefr-task-banner');
+    if (banner) {
+        banner.classList.add('hidden');
+        banner.innerHTML = '';
+    }
+}
+
 // -----------------------------------------------------------------------
 // Screen Switcher inside #cefr-view
 // -----------------------------------------------------------------------
@@ -108,6 +261,7 @@ function showCefrScreen(screenName) {
 // -----------------------------------------------------------------------
 export async function loadCefrOverview() {
     showCefrScreen('overview');
+    checkActiveMiningTask();
     const grid = document.getElementById('cefr-levels-grid');
     if (!grid) return;
 
@@ -312,6 +466,7 @@ function renderCurrentQuestion() {
         <div class="cefr-cloze-header">
             <span class="cefr-mode-tag">🧩 Cloze Tap (1-тап выбор)</span>
             ${q.pos ? `<span class="badge-pos">${escapeHtml(q.pos)}</span>` : ''}
+            ${q.source_book_title ? `<span class="cefr-book-badge" title="${escapeHtml(q.source_chapter_title || '')}">📖 «${escapeHtml(q.source_book_title)}»${q.source_chapter_title ? ` (${escapeHtml(q.source_chapter_title)})` : ''}</span>` : ''}
         </div>
 
         <div class="cefr-sentence-wrap">
@@ -383,6 +538,7 @@ async function handleOptionTap(questionId, selectedOption, clickedBtn) {
                         <strong class="fb-term">${escapeHtml(res.correct_option)}</strong>
                         ${res.phonetic ? `<span class="res-phonetic">${escapeHtml(res.phonetic)}</span>` : ''}
                     </div>
+                    ${res.source_book_title ? `<div class="fb-source-citation">📖 Пример из книги «${escapeHtml(res.source_book_title)}»${res.source_chapter_title ? ` (${escapeHtml(res.source_chapter_title)})` : ''}</div>` : ''}
                     ${res.explanation ? `<p class="fb-explanation">${escapeHtml(res.explanation)}</p>` : ''}
                     ${renderOptionsBreakdown(res.options_breakdown)}
                 `;
@@ -416,6 +572,7 @@ async function handleOptionTap(questionId, selectedOption, clickedBtn) {
                         <span class="fb-trans">${escapeHtml(res.translation || '')}</span>
                         ${res.phonetic ? `<span class="res-phonetic">${escapeHtml(res.phonetic)}</span>` : ''}
                     </div>
+                    ${res.source_book_title ? `<div class="fb-source-citation">📖 Пример из книги «${escapeHtml(res.source_book_title)}»${res.source_chapter_title ? ` (${escapeHtml(res.source_chapter_title)})` : ''}</div>` : ''}
                     ${res.explanation ? `<p class="fb-explanation">${escapeHtml(res.explanation)}</p>` : ''}
                     <div class="cefr-dual-ingested-box">
                         <div class="dual-ingested-header">

app/static/sw.js

@@ -1,7 +1,7 @@
 // Service Worker for Notes English PWA (Stage 2)
 // Strategy: Cache-first for static assets, Network-first for API calls.
 
-const CACHE_NAME = 'notes-v5.4';
+const CACHE_NAME = 'notes-v5.5';
 const OFFLINE_URL = '/';
 
 const PRECACHE_URLS = [

app/tasks.py

@@ -0,0 +1,295 @@
+"""
+Background Tasks & Async Job Manager.
+100% pure Python stdlib + SQLite WAL mode.
+Zero Redis, zero Celery, zero bloat (Ponytail architecture).
+"""
+
+import json
+import sqlite3
+import asyncio
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+
+# In-memory cancellation registry for instantaneous cooperative cancellation
+CANCELLED_TASK_IDS = set()
+
+
+def create_task(
+    conn: sqlite3.Connection,
+    user_id: int,
+    task_type: str,
+    params: Optional[Dict[str, Any]] = None,
+    total: int = 0,
+    message: str = "Задача поставлена в очередь",
+) -> int:
+    """
+    Creates a new background task in 'pending' status.
+    """
+    params_json = json.dumps(params or {}, ensure_ascii=False)
+    cursor = conn.execute(
+        """
+        INSERT INTO background_tasks (
+            user_id, task_type, status, progress_current, progress_total,
+            message, params_json, result_json, error_message,
+            created_at, updated_at
+        ) VALUES (?, ?, 'pending', 0, ?, ?, ?, '{}', '', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+        """,
+        (user_id, task_type, total, message, params_json),
+    )
+    return cursor.lastrowid
+
+
+def update_task_progress(
+    conn: sqlite3.Connection,
+    task_id: int,
+    current: int,
+    total: Optional[int] = None,
+    message: Optional[str] = None,
+):
+    """
+    Updates the progress and status of a running task.
+    """
+    if total is not None and message is not None:
+        conn.execute(
+            """
+            UPDATE background_tasks
+            SET status = 'running',
+                progress_current = ?,
+                progress_total = ?,
+                message = ?,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE id = ? AND status != 'cancelled'
+            """,
+            (current, total, message, task_id),
+        )
+    elif total is not None:
+        conn.execute(
+            """
+            UPDATE background_tasks
+            SET status = 'running',
+                progress_current = ?,
+                progress_total = ?,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE id = ? AND status != 'cancelled'
+            """,
+            (current, total, task_id),
+        )
+    elif message is not None:
+        conn.execute(
+            """
+            UPDATE background_tasks
+            SET status = 'running',
+                progress_current = ?,
+                message = ?,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE id = ? AND status != 'cancelled'
+            """,
+            (current, message, task_id),
+        )
+    else:
+        conn.execute(
+            """
+            UPDATE background_tasks
+            SET status = 'running',
+                progress_current = ?,
+                updated_at = CURRENT_TIMESTAMP
+            WHERE id = ? AND status != 'cancelled'
+            """,
+            (current, task_id),
+        )
+
+
+def complete_task(
+    conn: sqlite3.Connection,
+    task_id: int,
+    result: Optional[Dict[str, Any]] = None,
+    message: str = "Задача успешно завершена",
+):
+    """
+    Marks a background task as completed.
+    """
+    result_json = json.dumps(result or {}, ensure_ascii=False)
+    conn.execute(
+        """
+        UPDATE background_tasks
+        SET status = 'completed',
+            result_json = ?,
+            message = ?,
+            updated_at = CURRENT_TIMESTAMP
+        WHERE id = ? AND status != 'cancelled'
+        """,
+        (result_json, message, task_id),
+    )
+    CANCELLED_TASK_IDS.discard(task_id)
+
+
+def fail_task(
+    conn: sqlite3.Connection,
+    task_id: int,
+    error_message: str,
+):
+    """
+    Marks a background task as failed with an error message.
+    """
+    conn.execute(
+        """
+        UPDATE background_tasks
+        SET status = 'failed',
+            error_message = ?,
+            message = 'Ошибка выполнения задачи',
+            updated_at = CURRENT_TIMESTAMP
+        WHERE id = ? AND status != 'cancelled'
+        """,
+        (error_message, task_id),
+    )
+    CANCELLED_TASK_IDS.discard(task_id)
+
+
+def cancel_task(conn: sqlite3.Connection, task_id: int, user_id: int) -> bool:
+    """
+    Cooperatively cancels a task.
+    """
+    CANCELLED_TASK_IDS.add(task_id)
+    cursor = conn.execute(
+        """
+        UPDATE background_tasks
+        SET status = 'cancelled',
+            message = 'Задача отменена пользователем',
+            updated_at = CURRENT_TIMESTAMP
+        WHERE id = ? AND user_id = ? AND status IN ('pending', 'running')
+        """,
+        (task_id, user_id),
+    )
+    return cursor.rowcount > 0
+
+
+def is_task_cancelled(task_id: int, conn: Optional[sqlite3.Connection] = None) -> bool:
+    """
+    Checks if a task was requested to be cancelled.
+    """
+    if task_id in CANCELLED_TASK_IDS:
+        return True
+    if conn:
+        row = conn.execute(
+            "SELECT status FROM background_tasks WHERE id = ?", (task_id,)
+        ).fetchone()
+        if row and row[0] == "cancelled":
+            CANCELLED_TASK_IDS.add(task_id)
+            return True
+    return False
+
+
+def get_task_status(conn: sqlite3.Connection, task_id: int, user_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
+    """
+    Retrieves full details of a specific task.
+    """
+    if user_id is not None:
+        row = conn.execute(
+            """
+            SELECT id, user_id, task_type, status, progress_current, progress_total,
+                   message, params_json, result_json, error_message, created_at, updated_at
+            FROM background_tasks
+            WHERE id = ? AND user_id = ?
+            """,
+            (task_id, user_id),
+        ).fetchone()
+    else:
+        row = conn.execute(
+            """
+            SELECT id, user_id, task_type, status, progress_current, progress_total,
+                   message, params_json, result_json, error_message, created_at, updated_at
+            FROM background_tasks
+            WHERE id = ?
+            """,
+            (task_id,),
+        ).fetchone()
+
+    if not row:
+        return None
+
+    return _format_task_row(row)
+
+
+def get_active_task(conn: sqlite3.Connection, user_id: int, task_type: Optional[str] = None) -> Optional[Dict[str, Any]]:
+    """
+    Returns the most recent active (pending or running) task for user.
+    """
+    if task_type:
+        row = conn.execute(
+            """
+            SELECT id, user_id, task_type, status, progress_current, progress_total,
+                   message, params_json, result_json, error_message, created_at, updated_at
+            FROM background_tasks
+            WHERE user_id = ? AND status IN ('pending', 'running') AND task_type = ?
+            ORDER BY id DESC LIMIT 1
+            """,
+            (user_id, task_type),
+        ).fetchone()
+    else:
+        row = conn.execute(
+            """
+            SELECT id, user_id, task_type, status, progress_current, progress_total,
+                   message, params_json, result_json, error_message, created_at, updated_at
+            FROM background_tasks
+            WHERE user_id = ? AND status IN ('pending', 'running')
+            ORDER BY id DESC LIMIT 1
+            """,
+            (user_id,),
+        ).fetchone()
+
+    if not row:
+        return None
+
+    return _format_task_row(row)
+
+
+def list_user_tasks(conn: sqlite3.Connection, user_id: int, limit: int = 10) -> List[Dict[str, Any]]:
+    """
+    Returns recent background tasks for user.
+    """
+    rows = conn.execute(
+        """
+        SELECT id, user_id, task_type, status, progress_current, progress_total,
+               message, params_json, result_json, error_message, created_at, updated_at
+        FROM background_tasks
+        WHERE user_id = ?
+        ORDER BY id DESC LIMIT ?
+        """,
+        (user_id, limit),
+    ).fetchall()
+
+    return [_format_task_row(r) for r in rows]
+
+
+def _format_task_row(row: Any) -> Dict[str, Any]:
+    """Helper to convert sqlite3.Row or tuple to a clean JSON-friendly dict."""
+    params = {}
+    result = {}
+    try:
+        params = json.loads(row["params_json"] or "{}")
+    except Exception:
+        pass
+    try:
+        result = json.loads(row["result_json"] or "{}")
+    except Exception:
+        pass
+
+    total = row["progress_total"] or 0
+    current = row["progress_current"] or 0
+    percent = round((current / total) * 100) if total > 0 else 0
+
+    return {
+        "id": row["id"],
+        "user_id": row["user_id"],
+        "task_type": row["task_type"],
+        "status": row["status"],
+        "progress_current": current,
+        "progress_total": total,
+        "progress_percent": percent,
+        "message": row["message"] or "",
+        "params": params,
+        "result": result,
+        "error_message": row["error_message"] or "",
+        "created_at": str(row["created_at"]),
+        "updated_at": str(row["updated_at"]),
+    }

tests/test_api.py

@@ -1393,11 +1393,10 @@ def test_cefr_testing_and_gap_ingestion():
     assert "clue" in q1
     assert len(q1["options"]) == 4
 
-    # 5. Answer INCORRECTLY -> Automated Dual Gap Ingestion (target + selected option)
     # Pick a distractor from the question's actual options
-    # Find option that is not correct (we test with the first distractor)
+    with get_db() as conn:
-    # Let's inspect options and find one that is wrong
+        correct_term = conn.execute("SELECT term FROM cefr_questions WHERE id = ?", (q1["id"],)).fetchone()["term"]
-    wrong_opt = [opt for opt in q1["options"] if opt.lower() != q1.get("term", "").lower()][0]
+    wrong_opt = [opt for opt in q1["options"] if opt.lower() != correct_term.lower()][0]
 
     ans_wrong = client.post("/api/cefr/answer", json={
         "question_id": q1["id"],
@@ -1576,6 +1575,77 @@ def test_grammar_engine_and_handwritten_journaling():
     assert u1_after["handwritten_logged"] is True
 
 
+def test_background_tasks_and_book_mining():
+    # 1. Unauthenticated check
+    client.cookies.clear()
+    unauth = client.post("/api/tasks/mine-books", json={"level": "A1", "count": 2})
+    assert unauth.status_code == 401
+
+    # 2. Login
+    login_res = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
+    assert login_res.status_code == 200
+
+    # 3. Create a book with known vocabulary
+    book_resp = client.post("/api/books", json={
+        "title": "Adventures in London",
+        "author": "Arthur Conan Doyle",
+        "content": "Chapter 1: The Arrival\nThey decided to invite their neighbor for breakfast. The journey was long and dangerous.\n\nChapter 2: The Mystery\nHe could not remember where he left his expensive ticket.",
+        "source_type": "text"
+    })
+    assert book_resp.status_code == 200
+    book_id = book_resp.json()["id"]
+
+    # 4. Start mining task
+    mine_resp = client.post("/api/tasks/mine-books", json={
+        "book_id": book_id,
+        "level": "A1",
+        "count": 2
+    })
+    assert mine_resp.status_code == 200
+    data = mine_resp.json()
+    assert data["status"] == "ok"
+    assert "task_id" in data
+    task_id = data["task_id"]
+
+    # 5. Check task status via active and ID endpoint
+    active_resp = client.get("/api/tasks/active")
+    assert active_resp.status_code == 200
+
+    # Wait briefly or poll task status
+    import time
+    t_data = None
+    for _ in range(30):
+        t_res = client.get(f"/api/tasks/{task_id}")
+        assert t_res.status_code == 200
+        t_data = t_res.json()["task"]
+        if t_data["status"] in ("completed", "failed", "cancelled"):
+            break
+        time.sleep(0.1)
+
+    assert t_data is not None
+    assert t_data["status"] == "completed"
+    assert t_data["progress_current"] > 0
+
+    # 6. Verify quiz retrieval includes source book metadata
+    quiz_res = client.get("/api/cefr/quiz?level=A1&count=20")
+    assert quiz_res.status_code == 200
+    items = quiz_res.json()["quiz"]
+    book_items = [q for q in items if q.get("source_book_title") == "Adventures in London"]
+    assert len(book_items) > 0
+    assert book_items[0]["source_book_title"] == "Adventures in London"
+    assert "source_chapter_title" in book_items[0]
+
+    # 7. Test task cancellation
+    from app.tasks import create_task
+    with get_db() as conn:
+        task_id_pending = create_task(conn, 1, "mine_books", total=10)
+    cancel_resp = client.post(f"/api/tasks/{task_id_pending}/cancel")
+    assert cancel_resp.status_code == 200
+    assert cancel_resp.json()["status"] == "ok"
+    assert cancel_resp.json()["cancelled"] is True
+
+
+