feat(dict): high-speed offline dictionary with wordforms and zero daemon SQLite mmap

app/database.py

@@ -1,5 +1,6 @@
 import sqlite3
 import contextlib
+from pathlib import Path
 from typing import Generator
 from app import config
 
@@ -174,6 +175,59 @@ def _migrate(conn: sqlite3.Connection):
 
     conn.execute("CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id);")
 
+    # Ensure offline_dictionary table exists
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS offline_dictionary (
+        term TEXT PRIMARY KEY COLLATE NOCASE,
+        translation TEXT NOT NULL,
+        phonetic TEXT DEFAULT '',
+        pos TEXT DEFAULT ''
+    );
+    """)
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE);")
+
+    # Auto-seed offline dictionary if empty
+    _seed_offline_dictionary(conn)
+
+
+def _seed_offline_dictionary(conn: sqlite3.Connection):
+    """Seeds the offline dictionary from data/offline_dict.tsv.gz if the table is empty."""
+    try:
+        count = conn.execute("SELECT COUNT(*) FROM offline_dictionary").fetchone()[0]
+        if count > 0:
+            return
+    except Exception:
+        return
+
+    archive_candidates = [
+        config.BASE_DIR / "data" / "offline_dict.tsv.gz",
+        config.BASE_DIR / "app" / "data" / "offline_dict.tsv.gz",
+        Path("/app/data/offline_dict.tsv.gz"),
+    ]
+    archive_path = next((p for p in archive_candidates if p.is_file()), None)
+    if not archive_path:
+        return
+
+    import gzip
+
+    batch = []
+    with gzip.open(archive_path, "rt", encoding="utf-8") as f:
+        for line in f:
+            parts = line.rstrip("\r\n").split("\t")
+            if len(parts) == 4:
+                batch.append(parts)
+                if len(batch) >= 10000:
+                    conn.executemany(
+                        "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)",
+                        batch,
+                    )
+                    batch.clear()
+        if batch:
+            conn.executemany(
+                "INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, ?, ?)",
+                batch,
+            )
+
 
 def init_db():
     config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
@@ -271,6 +325,14 @@ def init_db():
         );
         CREATE INDEX IF NOT EXISTS idx_books_user ON books(user_id);
         CREATE INDEX IF NOT EXISTS idx_books_user_updated ON books(user_id, updated_at DESC);
+
+        CREATE TABLE IF NOT EXISTS offline_dictionary (
+            term TEXT PRIMARY KEY COLLATE NOCASE,
+            translation TEXT NOT NULL,
+            phonetic TEXT DEFAULT '',
+            pos TEXT DEFAULT ''
+        );
+        CREATE INDEX IF NOT EXISTS idx_offline_dict_term ON offline_dictionary(term COLLATE NOCASE);
         """)
         # Apply migrations for pre-existing databases
         _migrate(conn)