feat(dict): high-speed offline dictionary with wordforms and zero daemon SQLite mmap

This commit is contained in:
2026-10-06 12:01:08 +03:00
parent 9d4e911bfd
commit 9a5d588125
6 changed files with 602 additions and 15 deletions
+56 -3
View File
@@ -1142,14 +1142,67 @@ def test_debloat_dictionary_and_wiktionary_rest_parser():
res_defs = await fetch_wiktionary_definitions("reluctant", client=client)
assert "Adjective" in res_defs["parts_of_speech"]
assert len(res_defs["definitions"]) > 0
assert "<" not in res_defs["definitions"][0] and ">" not in res_defs["definitions"][0]
# 4. Test end-to-end lookup_term without Datamuse
term_res = await lookup_term("diligent")
assert term_res["query"] == "diligent"
assert len(term_res["translation"]) > 0
assert len(term_res["definitions"]) > 0
asyncio.run(run_checks())
def test_offline_dictionary_lookup_and_coverage():
"""Validates the Zero Daemon offline dictionary: 100k+ entries, word forms, instant mmap lookup."""
import asyncio
import time
from app.database import get_db
from app.dictionary import lookup_term
# 1. Verify offline_dictionary table exists and contains > 100,000 entries
with get_db() as conn:
count = conn.execute("SELECT COUNT(*) FROM offline_dictionary").fetchone()[0]
assert count > 100000, f"Expected > 100k entries, got {count}"
# Verify index exists
indexes = [r[1] for r in conn.execute("PRAGMA index_list('offline_dictionary')").fetchall()]
assert any("idx_offline_dict_term" in idx for idx in indexes)
# Check PRAGMA mmap_size
mmap_size = conn.execute("PRAGMA mmap_size").fetchone()[0]
assert mmap_size >= 268435456
async def run_lookups():
# 2. Test base words, inflected forms, irregulars, and IT terminology
test_words = [
("abandoned", "заброшенный"),
("ABANDONED", "заброшенный"), # Case-insensitivity
("hypotheses", "гипотез"), # Irregular plural
("men", "люди"), # Irregular plural
("went", "от go"), # Irregular past tense
("throughput", "пропускная способность"), # Networking / IT term
("quenched", "удовлетворить"), # Regular verb past tense (-ed)
]
for word, expected_snippet in test_words:
t0 = time.perf_counter()
res = await lookup_term(word)
dt_ms = (time.perf_counter() - t0) * 1000
assert dt_ms < 50.0, f"Lookup for {word} took {dt_ms:.2f}ms (expected < 50ms)"
assert res.get("source") == "offline" or res.get("translation"), f"Failed for {word}: {res}"
assert expected_snippet.lower() in res["translation"].lower(), f"Expected '{expected_snippet}' in '{res['translation']}'"
# 3. Test HTTP endpoint integration (/api/lookup)
login_res = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
assert login_res.status_code == 200
api_res = client.get("/api/lookup?term=throughput")
assert api_res.status_code == 200
json_data = api_res.json()
assert json_data["query"] == "throughput"
assert "пропускная способность" in json_data["translation"].lower()
assert json_data["source"] == "offline"
assert "in_library" in json_data
asyncio.run(run_lookups())