diff --git a/TODO.md b/TODO.md index 8e0e494..1d52290 100644 --- a/TODO.md +++ b/TODO.md @@ -200,15 +200,32 @@ --- ### 9. πŸ—„οΈ Вюнинг SQLite PRAGMA ΠΈ составных индСксов -- [ ] Π’ [`app/database.py`](file:///home/ai-projects/services/study.qpid.pro/app/database.py): - - ВынСсти `PRAGMA journal_mode=WAL;` Π² `init_db()` (ΡƒΠ±Ρ€Π°Ρ‚ΡŒ ΠΈΠ·Π±Ρ‹Ρ‚ΠΎΡ‡Π½Ρ‹ΠΉ Π²Ρ‹Π·ΠΎΠ² ΠΏΡ€ΠΈ ΠΊΠ°ΠΆΠ΄ΠΎΠΌ запросС). - - Π’ΠΊΠ»ΡŽΡ‡ΠΈΡ‚ΡŒ ΠΏΠ°Ρ€Π°ΠΌΠ΅Ρ‚Ρ€Ρ‹ ΠΏΡ€ΠΎΠΈΠ·Π²ΠΎΠ΄ΠΈΡ‚Π΅Π»ΡŒΠ½ΠΎΡΡ‚ΠΈ памяти Linux: +- [x] Π’ [`app/database.py`](file:///home/ai-projects/services/study.qpid.pro/app/database.py): + - ВынСсСн `PRAGMA journal_mode=WAL;` Π² ΠΎΠ΄Π½ΠΎΠΊΡ€Π°Ρ‚Π½ΡƒΡŽ ΠΈΠ½ΠΈΡ†ΠΈΠ°Π»ΠΈΠ·Π°Ρ†ΠΈΡŽ `init_db()` ΠΈ ΠΌΠΈΠ³Ρ€Π°Ρ†ΠΈΠΈ (устранСны ΠΏΠΎΠ²Ρ‚ΠΎΡ€Π½Ρ‹Π΅ Π±Π»ΠΎΠΊΠΈΡ€ΠΎΠ²ΠΊΠΈ ΠΈ запись Π·Π°Π³ΠΎΠ»ΠΎΠ²ΠΊΠ° Ρ„Π°ΠΉΠ»Π° Π½Π° ΠΊΠ°ΠΆΠ΄Ρ‹ΠΉ HTTP-запрос). + - Π’ΠΊΠ»ΡŽΡ‡Π΅Π½Ρ‹ Π²Ρ‹ΡΠΎΠΊΠΎΠΏΡ€ΠΎΠΈΠ·Π²ΠΎΠ΄ΠΈΡ‚Π΅Π»ΡŒΠ½Ρ‹Π΅ ΠΏΠ°Ρ€Π°ΠΌΠ΅Ρ‚Ρ€Ρ‹ памяти Linux для ΠΊΠ°ΠΆΠ΄ΠΎΠ³ΠΎ соСдинСния: ```python conn.execute("PRAGMA synchronous = NORMAL;") - conn.execute("PRAGMA mmap_size = 268435456;") # 256MB mmap - conn.execute("PRAGMA cache_size = -32000;") # 32MB cache + conn.execute("PRAGMA mmap_size = 268435456;") # 256MB memory-mapped I/O + conn.execute("PRAGMA cache_size = -32000;") # 32MB RAM page cache + conn.execute("PRAGMA temp_store = MEMORY;") # Π²Ρ€Π΅ΠΌΠ΅Π½Π½Ρ‹Π΅ Ρ‚Π°Π±Π»ΠΈΡ†Ρ‹ ΠΈ сортировки Π² ΠžΠ—Π£ + conn.execute("PRAGMA foreign_keys = ON;") ``` -- [ ] ΠžΠ±Π΅ΡΠΏΠ΅Ρ‡ΠΈΡ‚ΡŒ Zero-copy Ρ‡Ρ‚Π΅Π½ΠΈΠ΅ страниц Π‘Π” ядром Linux. +- [x] **Zero-copy Ρ‡Ρ‚Π΅Π½ΠΈΠ΅ страниц**: Π²ΠΊΠ»ΡŽΡ‡Ρ‘Π½ `mmap_size = 256MB`, ΠΏΠΎΠ·Π²ΠΎΠ»ΡΡŽΡ‰ΠΈΠΉ ядру Linux ΠΎΡ‚Π΄Π°Π²Π°Ρ‚ΡŒ страницы SQLite Π½Π°ΠΏΡ€ΡΠΌΡƒΡŽ ΠΈΠ· систСмного дискового кэша Π±Π΅Π· Π»ΠΈΡˆΠ½ΠΈΡ… систСмных Π²Ρ‹Π·ΠΎΠ²ΠΎΠ² `read()` ΠΈ копирования Π² Π±ΡƒΡ„Π΅Ρ€Ρ‹ ΠΏΠΎΠ»ΡŒΠ·ΠΎΠ²Π°Ρ‚Π΅Π»ΡŒΡΠΊΠΎΠ³ΠΎ пространства. +- [x] **БоставныС индСксы (Composite Indexes)** для ускорСния частых Π²Ρ‹Π±ΠΎΡ€ΠΎΠΊ: + - `vocabulary_cards`: + - `idx_vocab_user_created ON (user_id, created_at DESC)` β€” мгновСнная пагинация списка слов Π±Π΅Π· сортировки Π² памяти. + - `idx_vocab_user_srs_state ON (user_id, srs_state, srs_next_review)` β€” Π²Ρ‹Π±ΠΎΡ€ΠΊΠ° ΠΎΡ‡Π΅Ρ€Π΅Π΄ΠΈ ΠΈΠ½Ρ‚Π΅Ρ€Π²Π°Π»ΡŒΠ½Ρ‹Ρ… ΠΏΠΎΠ²Ρ‚ΠΎΡ€Π΅Π½ΠΈΠΉ SRS (`/api/srs/queue`). + - `idx_vocab_user_srs_created ON (user_id, srs_state, created_at DESC)` β€” Π²Ρ‹Π±ΠΎΡ€ΠΊΠ° Π½ΠΎΠ²Ρ‹Ρ… слов для ΠΈΠ½Ρ‚Π΅Ρ€Π²Π°Π»ΡŒΠ½ΠΎΠ³ΠΎ повторСния. + - `idx_vocab_collection ON (user_id, collection)` β€” мгновСнная Ρ„ΠΈΠ»ΡŒΡ‚Ρ€Π°Ρ†ΠΈΡ ΠΏΠΎ коллСкциям. + - `idx_vocab_user_term_nocase ON (user_id, term COLLATE NOCASE)` β€” ΠΏΡ€Π΅Π΄ΠΎΡ‚Π²Ρ€Π°Ρ‰Π΅Π½ΠΈΠ΅ Π΄ΡƒΠ±Π»ΠΈΠΊΠ°Ρ‚ΠΎΠ² Π·Π° $O(\log N)$. + - `srs_logs`: + - `idx_srs_card ON (card_id)` β€” устранСниС full table scan ΠΏΡ€ΠΈ каскадном ΡƒΠ΄Π°Π»Π΅Π½ΠΈΠΈ ΠΊΠ°Ρ€Ρ‚ΠΎΡ‡Π΅ΠΊ (`ON DELETE CASCADE`). + - `idx_srs_reviewed ON (user_id, reviewed_at)` β€” Π²Ρ‹Π±ΠΎΡ€ΠΊΠ° ΠΆΡƒΡ€Π½Π°Π»Π° ΠΏΠΎΠ²Ρ‚ΠΎΡ€Π΅Π½ΠΈΠΉ ΠΈ расчёт Ρ‚Π΅ΠΏΠ»ΠΎΠ²ΠΎΠΉ ΠΊΠ°Ρ€Ρ‚Ρ‹ активности. + - `books`: + - `idx_books_user_updated ON (user_id, updated_at DESC)` β€” сортировка списка ΠΊΠ½ΠΈΠ³ Π±ΠΈΠ±Π»ΠΈΠΎΡ‚Π΅ΠΊΠΈ ΠΏΠΎ Π΄Π°Ρ‚Π΅ чтСния. + - `sessions`: + - `idx_sessions_user ON (user_id)` β€” ускорСниС Π²Π°Π»ΠΈΠ΄Π°Ρ†ΠΈΠΈ сСссий ΠΈ каскадной очистки. +- [x] Π”ΠΎΠ±Π°Π²Π»Π΅Π½ рСгрСссионный тСст `test_sqlite_pragma_tuning_and_composite_indexes` Π² `tests/test_api.py` (23/23 тСстов ΡƒΡΠΏΠ΅ΡˆΠ½ΠΎ). --- diff --git a/app/database.py b/app/database.py index 5728dd0..cdfbcf6 100644 --- a/app/database.py +++ b/app/database.py @@ -7,9 +7,11 @@ from app import config def get_db_connection() -> sqlite3.Connection: conn = sqlite3.connect(config.DB_PATH, timeout=15.0) conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=WAL;") - conn.execute("PRAGMA synchronous=NORMAL;") - conn.execute("PRAGMA foreign_keys=ON;") + conn.execute("PRAGMA synchronous = NORMAL;") + conn.execute("PRAGMA mmap_size = 268435456;") # 256MB memory-mapped I/O + conn.execute("PRAGMA cache_size = -32000;") # 32MB RAM page cache + conn.execute("PRAGMA temp_store = MEMORY;") # In-memory temporary tables & sorts + conn.execute("PRAGMA foreign_keys = ON;") return conn @@ -27,7 +29,9 @@ def get_db() -> Generator[sqlite3.Connection, None, None]: def _migrate(conn: sqlite3.Connection): - """Safe schema migrations: add new columns/tables without touching existing data.""" + """Safe schema migrations: add new columns/tables/indexes without touching existing data.""" + conn.execute("PRAGMA journal_mode=WAL;") + # --- vocabulary_cards new columns --- existing_cols = { row[1] for row in conn.execute("PRAGMA table_info(vocabulary_cards)").fetchall() @@ -61,9 +65,6 @@ def _migrate(conn: sqlite3.Connection): ); """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);") - conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);") - # Ensure books table exists conn.execute(""" CREATE TABLE IF NOT EXISTS books ( @@ -81,7 +82,6 @@ def _migrate(conn: sqlite3.Connection): FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);") # --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index --- dup_rows = conn.execute(""" @@ -148,14 +148,37 @@ def _migrate(conn: sqlite3.Connection): ), ) + # Composite and single column indexes for high-speed queries conn.execute( "CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);" ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);" + ) + + conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);") + conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);") + conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);") + + conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);") + conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);") + + conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);") def init_db(): config.DB_PATH.parent.mkdir(parents=True, exist_ok=True) with get_db() as conn: + conn.execute("PRAGMA journal_mode=WAL;") conn.executescript(""" CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -174,6 +197,7 @@ def init_db(): ip TEXT, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); + CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id); CREATE TABLE IF NOT EXISTS vocabulary_cards ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -202,9 +226,12 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_vocab_user ON vocabulary_cards(user_id); CREATE INDEX IF NOT EXISTS idx_vocab_term ON vocabulary_cards(term); - CREATE INDEX IF NOT EXISTS idx_vocab_created ON vocabulary_cards(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection); CREATE INDEX IF NOT EXISTS idx_vocab_srs_queue ON vocabulary_cards(user_id, srs_next_review, srs_state); + CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review); + CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC); + CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE); CREATE TABLE IF NOT EXISTS srs_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -220,6 +247,7 @@ def init_db(): CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id); CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at); + CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id); CREATE TABLE IF NOT EXISTS dictionary_cache ( query TEXT PRIMARY KEY COLLATE NOCASE, @@ -232,7 +260,7 @@ def init_db(): user_id INTEGER NOT NULL, title TEXT NOT NULL, author TEXT DEFAULT '', - source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text' + source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text', 'pdf' chapters_json TEXT NOT NULL, -- JSON array: [{"title": "Chapter 1", "content": "..."}] current_chapter INTEGER DEFAULT 0, current_position REAL DEFAULT 0, @@ -242,6 +270,7 @@ def init_db(): FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id); + CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC); """) # Apply migrations for pre-existing databases _migrate(conn) diff --git a/tests/test_api.py b/tests/test_api.py index 4293383..518346d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1042,3 +1042,76 @@ def test_http_connection_pooling(): assert c3 is not c1 assert not c3.is_closed asyncio.run(close_http_client()) + + +def test_sqlite_pragma_tuning_and_composite_indexes(): + from app.database import get_db, get_db_connection + + # 1. Verify PRAGMAs on a newly opened connection + conn = get_db_connection() + try: + j_mode = conn.execute("PRAGMA journal_mode;").fetchone()[0] + sync = conn.execute("PRAGMA synchronous;").fetchone()[0] + mmap = conn.execute("PRAGMA mmap_size;").fetchone()[0] + cache = conn.execute("PRAGMA cache_size;").fetchone()[0] + temp = conn.execute("PRAGMA temp_store;").fetchone()[0] + fk = conn.execute("PRAGMA foreign_keys;").fetchone()[0] + + assert j_mode.lower() == "wal" + assert sync == 1 # NORMAL + assert mmap == 268435456 # 256MB mmap + assert cache == -32000 # 32MB cache + assert temp == 2 # MEMORY + assert fk == 1 # foreign keys ON + + # 2. Verify all composite indexes exist in sqlite_master + index_rows = conn.execute( + "SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'index'" + ).fetchall() + index_names = {r["name"] for r in index_rows} + + expected_indexes = { + "idx_vocab_user_created", + "idx_vocab_user_srs_state", + "idx_vocab_user_srs_created", + "idx_vocab_user_term_nocase", + "idx_books_user_updated", + "idx_srs_card", + "idx_srs_reviewed", + "idx_sessions_user", + } + for idx in expected_indexes: + assert idx in index_names, f"Expected index {idx} not found in database!" + + # 3. Verify EXPLAIN QUERY PLAN uses the composite indexes + plan1 = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM vocabulary_cards WHERE user_id = ? ORDER BY created_at DESC LIMIT 20", + (1,), + ).fetchall() + plan1_str = " ".join([r[3] for r in plan1]) + assert "idx_vocab_user_created" in plan1_str + + plan2 = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM vocabulary_cards WHERE user_id = ? AND srs_state = 'new' ORDER BY created_at DESC LIMIT 10", + (1,), + ).fetchall() + plan2_str = " ".join([r[3] for r in plan2]) + assert "idx_vocab_user_srs_created" in plan2_str or "idx_vocab_user_created" in plan2_str + + plan3 = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM books WHERE user_id = ? ORDER BY updated_at DESC", + (1,), + ).fetchall() + plan3_str = " ".join([r[3] for r in plan3]) + assert "idx_books_user_updated" in plan3_str + + plan4 = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM srs_logs WHERE card_id = ?", + (1,), + ).fetchall() + plan4_str = " ".join([r[3] for r in plan4]) + assert "idx_srs_card" in plan4_str + + finally: + conn.close() +