feat(dict): high-speed offline dictionary with wordforms and zero daemon SQLite mmap
This commit is contained in:
+56
-3
@@ -1142,14 +1142,67 @@ def test_debloat_dictionary_and_wiktionary_rest_parser():
|
||||
res_defs = await fetch_wiktionary_definitions("reluctant", client=client)
|
||||
assert "Adjective" in res_defs["parts_of_speech"]
|
||||
assert len(res_defs["definitions"]) > 0
|
||||
assert "<" not in res_defs["definitions"][0] and ">" not in res_defs["definitions"][0]
|
||||
|
||||
# 4. Test end-to-end lookup_term without Datamuse
|
||||
term_res = await lookup_term("diligent")
|
||||
assert term_res["query"] == "diligent"
|
||||
assert len(term_res["translation"]) > 0
|
||||
assert len(term_res["definitions"]) > 0
|
||||
|
||||
asyncio.run(run_checks())
|
||||
|
||||
|
||||
def test_offline_dictionary_lookup_and_coverage():
|
||||
"""Validates the Zero Daemon offline dictionary: 100k+ entries, word forms, instant mmap lookup."""
|
||||
import asyncio
|
||||
import time
|
||||
from app.database import get_db
|
||||
from app.dictionary import lookup_term
|
||||
|
||||
# 1. Verify offline_dictionary table exists and contains > 100,000 entries
|
||||
with get_db() as conn:
|
||||
count = conn.execute("SELECT COUNT(*) FROM offline_dictionary").fetchone()[0]
|
||||
assert count > 100000, f"Expected > 100k entries, got {count}"
|
||||
|
||||
# Verify index exists
|
||||
indexes = [r[1] for r in conn.execute("PRAGMA index_list('offline_dictionary')").fetchall()]
|
||||
assert any("idx_offline_dict_term" in idx for idx in indexes)
|
||||
|
||||
# Check PRAGMA mmap_size
|
||||
mmap_size = conn.execute("PRAGMA mmap_size").fetchone()[0]
|
||||
assert mmap_size >= 268435456
|
||||
|
||||
async def run_lookups():
|
||||
# 2. Test base words, inflected forms, irregulars, and IT terminology
|
||||
test_words = [
|
||||
("abandoned", "заброшенный"),
|
||||
("ABANDONED", "заброшенный"), # Case-insensitivity
|
||||
("hypotheses", "гипотез"), # Irregular plural
|
||||
("men", "люди"), # Irregular plural
|
||||
("went", "от go"), # Irregular past tense
|
||||
("throughput", "пропускная способность"), # Networking / IT term
|
||||
("quenched", "удовлетворить"), # Regular verb past tense (-ed)
|
||||
]
|
||||
|
||||
for word, expected_snippet in test_words:
|
||||
t0 = time.perf_counter()
|
||||
res = await lookup_term(word)
|
||||
dt_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
assert dt_ms < 50.0, f"Lookup for {word} took {dt_ms:.2f}ms (expected < 50ms)"
|
||||
assert res.get("source") == "offline" or res.get("translation"), f"Failed for {word}: {res}"
|
||||
assert expected_snippet.lower() in res["translation"].lower(), f"Expected '{expected_snippet}' in '{res['translation']}'"
|
||||
|
||||
# 3. Test HTTP endpoint integration (/api/lookup)
|
||||
login_res = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
|
||||
assert login_res.status_code == 200
|
||||
|
||||
api_res = client.get("/api/lookup?term=throughput")
|
||||
assert api_res.status_code == 200
|
||||
json_data = api_res.json()
|
||||
assert json_data["query"] == "throughput"
|
||||
assert "пропускная способность" in json_data["translation"].lower()
|
||||
assert json_data["source"] == "offline"
|
||||
assert "in_library" in json_data
|
||||
|
||||
asyncio.run(run_lookups())
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user