feat: optimize SQLite PRAGMAs (WAL, mmap 256MB, cache 32MB, temp_store) and add compound indexes

TODO.md

@@ -200,15 +200,32 @@
 ---
 
 ### 9. 🗄️ Тюнинг SQLite PRAGMA и составных индексов
-- [ ] В [`app/database.py`](file:///home/ai-projects/services/study.qpid.pro/app/database.py):
+- [x] В [`app/database.py`](file:///home/ai-projects/services/study.qpid.pro/app/database.py):
-  - Вынести `PRAGMA journal_mode=WAL;` в `init_db()` (убрать избыточный вызов при каждом запросе).
+  - Вынесен `PRAGMA journal_mode=WAL;` в однократную инициализацию `init_db()` и миграции (устранены повторные блокировки и запись заголовка файла на каждый HTTP-запрос).
-  - Включить параметры производительности памяти Linux:
+  - Включены высокопроизводительные параметры памяти Linux для каждого соединения:
     ```python
     conn.execute("PRAGMA synchronous = NORMAL;")
-    conn.execute("PRAGMA mmap_size = 268435456;") # 256MB mmap
+    conn.execute("PRAGMA mmap_size = 268435456;")  # 256MB memory-mapped I/O
-    conn.execute("PRAGMA cache_size = -32000;")   # 32MB cache
+    conn.execute("PRAGMA cache_size = -32000;")     # 32MB RAM page cache
+    conn.execute("PRAGMA temp_store = MEMORY;")     # временные таблицы и сортировки в ОЗУ
+    conn.execute("PRAGMA foreign_keys = ON;")
     ```
-- [ ] Обеспечить Zero-copy чтение страниц БД ядром Linux.
+- [x] **Zero-copy чтение страниц**: включён `mmap_size = 256MB`, позволяющий ядру Linux отдавать страницы SQLite напрямую из системного дискового кэша без лишних системных вызовов `read()` и копирования в буферы пользовательского пространства.
+- [x] **Составные индексы (Composite Indexes)** для ускорения частых выборок:
+  - `vocabulary_cards`:
+    - `idx_vocab_user_created ON (user_id, created_at DESC)` — мгновенная пагинация списка слов без сортировки в памяти.
+    - `idx_vocab_user_srs_state ON (user_id, srs_state, srs_next_review)` — выборка очереди интервальных повторений SRS (`/api/srs/queue`).
+    - `idx_vocab_user_srs_created ON (user_id, srs_state, created_at DESC)` — выборка новых слов для интервального повторения.
+    - `idx_vocab_collection ON (user_id, collection)` — мгновенная фильтрация по коллекциям.
+    - `idx_vocab_user_term_nocase ON (user_id, term COLLATE NOCASE)` — предотвращение дубликатов за $O(\log N)$.
+  - `srs_logs`:
+    - `idx_srs_card ON (card_id)` — устранение full table scan при каскадном удалении карточек (`ON DELETE CASCADE`).
+    - `idx_srs_reviewed ON (user_id, reviewed_at)` — выборка журнала повторений и расчёт тепловой карты активности.
+  - `books`:
+    - `idx_books_user_updated ON (user_id, updated_at DESC)` — сортировка списка книг библиотеки по дате чтения.
+  - `sessions`:
+    - `idx_sessions_user ON (user_id)` — ускорение валидации сессий и каскадной очистки.
+- [x] Добавлен регрессионный тест `test_sqlite_pragma_tuning_and_composite_indexes` в `tests/test_api.py` (23/23 тестов успешно).
 
 ---
 

app/database.py

@@ -7,8 +7,10 @@ from app import config
 def get_db_connection() -> sqlite3.Connection:
     conn = sqlite3.connect(config.DB_PATH, timeout=15.0)
     conn.row_factory = sqlite3.Row
-    conn.execute("PRAGMA journal_mode=WAL;")
     conn.execute("PRAGMA synchronous = NORMAL;")
+    conn.execute("PRAGMA mmap_size = 268435456;")  # 256MB memory-mapped I/O
+    conn.execute("PRAGMA cache_size = -32000;")     # 32MB RAM page cache
+    conn.execute("PRAGMA temp_store = MEMORY;")     # In-memory temporary tables & sorts
     conn.execute("PRAGMA foreign_keys = ON;")
     return conn
 
@@ -27,7 +29,9 @@ def get_db() -> Generator[sqlite3.Connection, None, None]:
 
 
 def _migrate(conn: sqlite3.Connection):
-    """Safe schema migrations: add new columns/tables without touching existing data."""
+    """Safe schema migrations: add new columns/tables/indexes without touching existing data."""
+    conn.execute("PRAGMA journal_mode=WAL;")
+
     # --- vocabulary_cards new columns ---
     existing_cols = {
         row[1] for row in conn.execute("PRAGMA table_info(vocabulary_cards)").fetchall()
@@ -61,9 +65,6 @@ def _migrate(conn: sqlite3.Connection):
     );
     """)
 
-    conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);")
-    conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);")
-
     # Ensure books table exists
     conn.execute("""
     CREATE TABLE IF NOT EXISTS books (
@@ -81,7 +82,6 @@ def _migrate(conn: sqlite3.Connection):
         FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
     );
     """)
-    conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);")
 
     # --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index ---
     dup_rows = conn.execute("""
@@ -148,14 +148,37 @@ def _migrate(conn: sqlite3.Connection):
             ),
         )
 
+    # Composite and single column indexes for high-speed queries
     conn.execute(
         "CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);"
     )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);"
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);"
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);"
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);"
+    )
+
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);")
+
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);")
+
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);")
 
 
 def init_db():
     config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
     with get_db() as conn:
+        conn.execute("PRAGMA journal_mode=WAL;")
         conn.executescript("""
         CREATE TABLE IF NOT EXISTS users (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -174,6 +197,7 @@ def init_db():
             ip TEXT,
             FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
         );
+        CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);
 
         CREATE TABLE IF NOT EXISTS vocabulary_cards (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -202,9 +226,12 @@ def init_db():
 
         CREATE INDEX IF NOT EXISTS idx_vocab_user ON vocabulary_cards(user_id);
         CREATE INDEX IF NOT EXISTS idx_vocab_term ON vocabulary_cards(term);
-        CREATE INDEX IF NOT EXISTS idx_vocab_created ON vocabulary_cards(created_at DESC);
+        CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);
         CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);
         CREATE INDEX IF NOT EXISTS idx_vocab_srs_queue ON vocabulary_cards(user_id, srs_next_review, srs_state);
+        CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);
+        CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);
 
         CREATE TABLE IF NOT EXISTS srs_logs (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -220,6 +247,7 @@ def init_db():
 
         CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);
         CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);
+        CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);
 
         CREATE TABLE IF NOT EXISTS dictionary_cache (
             query TEXT PRIMARY KEY COLLATE NOCASE,
@@ -232,7 +260,7 @@ def init_db():
             user_id INTEGER NOT NULL,
             title TEXT NOT NULL,
             author TEXT DEFAULT '',
-            source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text'
+            source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text', 'pdf'
             chapters_json TEXT NOT NULL, -- JSON array: [{"title": "Chapter 1", "content": "..."}]
             current_chapter INTEGER DEFAULT 0,
             current_position REAL DEFAULT 0,
@@ -242,6 +270,7 @@ def init_db():
             FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
         );
         CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);
+        CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);
         """)
         # Apply migrations for pre-existing databases
         _migrate(conn)

tests/test_api.py

@@ -1042,3 +1042,76 @@ def test_http_connection_pooling():
     assert c3 is not c1
     assert not c3.is_closed
     asyncio.run(close_http_client())
+
+
+def test_sqlite_pragma_tuning_and_composite_indexes():
+    from app.database import get_db, get_db_connection
+
+    # 1. Verify PRAGMAs on a newly opened connection
+    conn = get_db_connection()
+    try:
+        j_mode = conn.execute("PRAGMA journal_mode;").fetchone()[0]
+        sync = conn.execute("PRAGMA synchronous;").fetchone()[0]
+        mmap = conn.execute("PRAGMA mmap_size;").fetchone()[0]
+        cache = conn.execute("PRAGMA cache_size;").fetchone()[0]
+        temp = conn.execute("PRAGMA temp_store;").fetchone()[0]
+        fk = conn.execute("PRAGMA foreign_keys;").fetchone()[0]
+
+        assert j_mode.lower() == "wal"
+        assert sync == 1  # NORMAL
+        assert mmap == 268435456  # 256MB mmap
+        assert cache == -32000  # 32MB cache
+        assert temp == 2  # MEMORY
+        assert fk == 1  # foreign keys ON
+
+        # 2. Verify all composite indexes exist in sqlite_master
+        index_rows = conn.execute(
+            "SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'index'"
+        ).fetchall()
+        index_names = {r["name"] for r in index_rows}
+
+        expected_indexes = {
+            "idx_vocab_user_created",
+            "idx_vocab_user_srs_state",
+            "idx_vocab_user_srs_created",
+            "idx_vocab_user_term_nocase",
+            "idx_books_user_updated",
+            "idx_srs_card",
+            "idx_srs_reviewed",
+            "idx_sessions_user",
+        }
+        for idx in expected_indexes:
+            assert idx in index_names, f"Expected index {idx} not found in database!"
+
+        # 3. Verify EXPLAIN QUERY PLAN uses the composite indexes
+        plan1 = conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM vocabulary_cards WHERE user_id = ? ORDER BY created_at DESC LIMIT 20",
+            (1,),
+        ).fetchall()
+        plan1_str = " ".join([r[3] for r in plan1])
+        assert "idx_vocab_user_created" in plan1_str
+
+        plan2 = conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM vocabulary_cards WHERE user_id = ? AND srs_state = 'new' ORDER BY created_at DESC LIMIT 10",
+            (1,),
+        ).fetchall()
+        plan2_str = " ".join([r[3] for r in plan2])
+        assert "idx_vocab_user_srs_created" in plan2_str or "idx_vocab_user_created" in plan2_str
+
+        plan3 = conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM books WHERE user_id = ? ORDER BY updated_at DESC",
+            (1,),
+        ).fetchall()
+        plan3_str = " ".join([r[3] for r in plan3])
+        assert "idx_books_user_updated" in plan3_str
+
+        plan4 = conn.execute(
+            "EXPLAIN QUERY PLAN SELECT * FROM srs_logs WHERE card_id = ?",
+            (1,),
+        ).fetchall()
+        plan4_str = " ".join([r[3] for r in plan4])
+        assert "idx_srs_card" in plan4_str
+
+    finally:
+        conn.close()
+