Antigravity/notes-qpid-pro
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: optimize SQLite PRAGMAs (WAL, mmap 256MB, cache 32MB, temp_store) and add compound indexes

Browse Source
This commit is contained in:
Antigravity
2026-10-06 11:38:46 +03:00
parent 438dfdb055
commit 078fa08fbd
3 changed files with 135 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/database.py
+39 -10
View File
@@ -7,9 +7,11 @@ from app import config
def get_db_connection() -> sqlite3.Connection:
conn = sqlite3.connect(config.DB_PATH, timeout=15.0)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL;")
conn.execute("PRAGMA synchronous=NORMAL;")
conn.execute("PRAGMA foreign_keys=ON;")
conn.execute("PRAGMA synchronous = NORMAL;")
conn.execute("PRAGMA mmap_size = 268435456;") # 256MB memory-mapped I/O
conn.execute("PRAGMA cache_size = -32000;") # 32MB RAM page cache
conn.execute("PRAGMA temp_store = MEMORY;") # In-memory temporary tables & sorts
conn.execute("PRAGMA foreign_keys = ON;")
return conn
@@ -27,7 +29,9 @@ def get_db() -> Generator[sqlite3.Connection, None, None]:
def _migrate(conn: sqlite3.Connection):
"""Safe schema migrations: add new columns/tables without touching existing data."""
"""Safe schema migrations: add new columns/tables/indexes without touching existing data."""
conn.execute("PRAGMA journal_mode=WAL;")
# --- vocabulary_cards new columns ---
existing_cols = {
row[1] for row in conn.execute("PRAGMA table_info(vocabulary_cards)").fetchall()
@@ -61,9 +65,6 @@ def _migrate(conn: sqlite3.Connection):
);
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);")
# Ensure books table exists
conn.execute("""
CREATE TABLE IF NOT EXISTS books (
@@ -81,7 +82,6 @@ def _migrate(conn: sqlite3.Connection):
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
);
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);")
# --- Deduplicate vocabulary_cards (case-insensitive) and enforce UNIQUE index ---
dup_rows = conn.execute("""
@@ -148,14 +148,37 @@ def _migrate(conn: sqlite3.Connection):
),
)
# Composite and single column indexes for high-speed queries
conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);")
conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);")
def init_db():
config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
with get_db() as conn:
conn.execute("PRAGMA journal_mode=WAL;")
conn.executescript("""
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -174,6 +197,7 @@ def init_db():
ip TEXT,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);
CREATE TABLE IF NOT EXISTS vocabulary_cards (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -202,9 +226,12 @@ def init_db():
CREATE INDEX IF NOT EXISTS idx_vocab_user ON vocabulary_cards(user_id);
CREATE INDEX IF NOT EXISTS idx_vocab_term ON vocabulary_cards(term);
CREATE INDEX IF NOT EXISTS idx_vocab_created ON vocabulary_cards(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_vocab_user_created ON vocabulary_cards(user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_vocab_collection ON vocabulary_cards(user_id, collection);
CREATE INDEX IF NOT EXISTS idx_vocab_srs_queue ON vocabulary_cards(user_id, srs_next_review, srs_state);
CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_state ON vocabulary_cards(user_id, srs_state, srs_next_review);
CREATE INDEX IF NOT EXISTS idx_vocab_user_srs_created ON vocabulary_cards(user_id, srs_state, created_at DESC);
CREATE UNIQUE INDEX IF NOT EXISTS idx_vocab_user_term_nocase ON vocabulary_cards(user_id, term COLLATE NOCASE);
CREATE TABLE IF NOT EXISTS srs_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -220,6 +247,7 @@ def init_db():
CREATE INDEX IF NOT EXISTS idx_srs_user ON srs_logs(user_id);
CREATE INDEX IF NOT EXISTS idx_srs_reviewed ON srs_logs(user_id, reviewed_at);
CREATE INDEX IF NOT EXISTS idx_srs_card ON srs_logs(card_id);
CREATE TABLE IF NOT EXISTS dictionary_cache (
query TEXT PRIMARY KEY COLLATE NOCASE,
@@ -232,7 +260,7 @@ def init_db():
user_id INTEGER NOT NULL,
title TEXT NOT NULL,
author TEXT DEFAULT '',
source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text'
source_type TEXT NOT NULL, -- 'epub', 'txt', 'md', 'text', 'pdf'
chapters_json TEXT NOT NULL, -- JSON array: [{"title": "Chapter 1", "content": "..."}]
current_chapter INTEGER DEFAULT 0,
current_position REAL DEFAULT 0,
@@ -242,6 +270,7 @@ def init_db():
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);
CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);
""")
# Apply migrations for pre-existing databases
_migrate(conn)