Antigravity/notes-qpid-pro
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: debloat dictionary parsers (remove Datamuse, switch to Wiktionary REST API with English section isolation)

Browse Source
This commit is contained in:
Antigravity
2026-10-06 11:46:51 +03:00
parent 078fa08fbd
commit 9d4e911bfd
3 changed files with 187 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/dictionary.py
+118 -105
View File
@@ -1,5 +1,6 @@
import json
import re
import html
import urllib.parse
from typing import Optional, Dict, Any, List
import httpx
@@ -14,7 +15,7 @@ _http_client: Optional[httpx.AsyncClient] = None
def get_http_client() -> httpx.AsyncClient:
"""
Returns a shared singleton httpx.AsyncClient instance with connection pooling and Keep-Alive.
Reuses persistent TCP/TLS connections to dictionary providers (Wiktionary, Datamuse, MyMemory)
Reuses persistent TCP/TLS connections to dictionary providers (Wiktionary, MyMemory)
eliminating 300-600ms of repeated connection handshakes per word lookup.
"""
global _http_client
@@ -35,120 +36,139 @@ async def close_http_client():
Closes the shared singleton HTTP client on application shutdown.
"""
global _http_client
if _http_client is not None and not _http_client.is_closed:
await _http_client.aclose()
if _http_client is not None:
client = _http_client
_http_client = None
if not client.is_closed:
try:
await client.aclose()
except Exception:
pass
async def fetch_wiktionary_wikitext(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]:
url = "https://en.wiktionary.org/w/api.php"
params = {
"action": "parse",
"page": word,
"prop": "wikitext",
"format": "json"
}
try:
http = client or get_http_client()
resp = await http.get(url, params=params, timeout=4.0)
if resp.status_code == 200:
data = resp.json()
wikitext = data.get("parse", {}).get("wikitext", {}).get("*", "")
"""
Fetches wikitext via Wiktionary modern REST API (fallback to action=parse)
and extracts clean IPA phonetics and Russian translations from the English section.
"""
http = client or get_http_client()
words_to_try = [word] if word == word.lower() else [word, word.lower()]
# Extract IPA
ipa_raw = re.findall(r"\{\{IPA\|en\|([^}]+)\}\}", wikitext)
ipa_cleaned = []
for entry in ipa_raw:
parts = entry.split("|")
for p in parts:
if p.startswith("/"):
for w in words_to_try:
wikitext = ""
# 1. Try modern Wikimedia REST API (faster, lightweight source)
url_rest = f"https://en.wiktionary.org/w/rest.php/v1/page/{urllib.parse.quote(w)}"
try:
resp = await http.get(url_rest, timeout=4.0)
if resp.status_code == 200:
data = resp.json()
wikitext = data.get("source", "")
except Exception:
pass
# 2. Fallback to legacy action=parse if REST returned empty
if not wikitext:
url_legacy = "https://en.wiktionary.org/w/api.php"
params = {
"action": "parse",
"page": w,
"prop": "wikitext",
"format": "json"
}
try:
resp = await http.get(url_legacy, params=params, timeout=4.0)
if resp.status_code == 200:
data = resp.json()
wikitext = data.get("parse", {}).get("wikitext", {}).get("*", "")
except Exception:
pass
if not wikitext:
continue
# Isolate English section to avoid leakage from other languages on the page
en_match = re.search(r"==English==\s*(.*?)(?=\n==[A-Za-z]|\Z)", wikitext, re.DOTALL)
section = en_match.group(1) if en_match else wikitext
# Extract IPA
ipa_raw = re.findall(r"\{\{IPA\|en\|([^}]+)\}\}", section)
ipa_cleaned = []
for entry in ipa_raw:
for p in entry.split("|"):
p = p.strip()
if p.startswith("/") and p.endswith("/"):
if p not in ipa_cleaned:
ipa_cleaned.append(p)
elif p.startswith("/"):
p_slash = p + "/"
if p_slash not in ipa_cleaned:
ipa_cleaned.append(p_slash)
elif p.startswith("[") and p.endswith("]"):
if p not in ipa_cleaned:
ipa_cleaned.append(p)
# Extract Russian translations
ru_raw = re.findall(r"\{\{t[+-]?\|ru\|([^|}]+)", wikitext)
ru_cleaned = []
for r in ru_raw:
clean = r.replace("[[", "").replace("]]", "").strip()
if clean and clean not in ru_cleaned:
ru_cleaned.append(clean)
# Extract Russian translations
ru_raw = re.findall(r"\{\{t[+-]?\|ru\|([^|}]+)", section)
ru_cleaned = []
for r in ru_raw:
clean = r.replace("[[", "").replace("]]", "").replace("́", "").strip()
if clean and clean not in ru_cleaned:
ru_cleaned.append(clean)
if ipa_cleaned or ru_cleaned:
return {
"ipa": ipa_cleaned,
"audio": [],
"ru_translations": ru_cleaned
}
except Exception:
pass
return {"ipa": [], "audio": [], "ru_translations": []}
async def fetch_wiktionary_definitions(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]:
url = f"https://en.wiktionary.org/api/rest_v1/page/definition/{urllib.parse.quote(word)}"
try:
http = client or get_http_client()
resp = await http.get(url, timeout=4.0)
if resp.status_code == 200:
data = resp.json()
en_entries = data.get("en", [])
parts_of_speech = []
definitions = []
examples = []
for entry in en_entries:
pos = entry.get("partOfSpeech")
if pos and pos not in parts_of_speech:
parts_of_speech.append(pos)
for d in entry.get("definitions", []):
raw_def = d.get("definition", "")
clean_def = re.sub(r"<[^>]+>", "", raw_def).strip()
if clean_def and clean_def not in definitions:
definitions.append(clean_def)
for ex in d.get("examples", []):
clean_ex = re.sub(r"<[^>]+>", "", ex).strip()
if clean_ex and clean_ex not in examples:
examples.append(clean_ex)
return {
"parts_of_speech": parts_of_speech,
"definitions": definitions[:4],
"examples": examples[:3]
}
except Exception:
pass
return {"parts_of_speech": [], "definitions": [], "examples": []}
async def fetch_datamuse(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]:
url = "https://api.datamuse.com/words"
params = {"sp": word, "md": "drp", "ipa": "1"}
try:
http = client or get_http_client()
resp = await http.get(url, params=params, timeout=3.0)
if resp.status_code == 200:
items = resp.json()
if items and isinstance(items, list):
first = items[0]
tags = first.get("tags", [])
ipa = None
pos = None
for t in tags:
if t.startswith("ipa_pron:"):
ipa = "/" + t.split(":", 1)[1] + "/"
elif t in ("n", "v", "adj", "adv"):
pos_map = {"n": "Noun", "v": "Verb", "adj": "Adjective", "adv": "Adverb"}
pos = pos_map.get(t)
defs = []
for d in first.get("defs", []):
parts = d.split("\t", 1)
defs.append(parts[1].strip() if len(parts) > 1 else parts[0].strip())
"""
Fetches structured definitions from Wiktionary REST endpoint
(https://en.wiktionary.org/api/rest_v1/page/definition/{word}) with HTML unescaping and whitespace normalization.
"""
http = client or get_http_client()
words_to_try = [word] if word == word.lower() else [word, word.lower()]
for w in words_to_try:
url = f"https://en.wiktionary.org/api/rest_v1/page/definition/{urllib.parse.quote(w)}"
try:
resp = await http.get(url, timeout=4.0)
if resp.status_code == 200:
data = resp.json()
en_entries = data.get("en", [])
if not en_entries:
continue
parts_of_speech = []
definitions = []
examples = []
for entry in en_entries:
pos = entry.get("partOfSpeech")
if pos and pos not in parts_of_speech:
parts_of_speech.append(pos)
for d in entry.get("definitions", []):
raw_def = d.get("definition", "")
clean_def = html.unescape(re.sub(r"<[^>]+>", "", raw_def)).strip()
clean_def = re.sub(r"\s+", " ", clean_def).strip()
if clean_def and clean_def not in definitions:
definitions.append(clean_def)
for ex in d.get("examples", []):
clean_ex = html.unescape(re.sub(r"<[^>]+>", "", ex)).strip()
clean_ex = re.sub(r"\s+", " ", clean_ex).strip()
if clean_ex and clean_ex not in examples:
examples.append(clean_ex)
return {
"ipa": ipa,
"part_of_speech": pos,
"definitions": defs[:3]
"parts_of_speech": parts_of_speech,
"definitions": definitions[:4],
"examples": examples[:3]
}
except Exception:
pass
return {"ipa": None, "part_of_speech": None, "definitions": []}
except Exception:
pass
return {"parts_of_speech": [], "definitions": [], "examples": []}
async def fetch_mymemory(query: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]:
@@ -251,35 +271,28 @@ async def lookup_term(query: str) -> Dict[str, Any]:
tasks = [
fetch_wiktionary_wikitext(query_clean, client=http_client),
fetch_wiktionary_definitions(query_clean, client=http_client),
fetch_datamuse(query_clean, client=http_client),
fetch_mymemory(query_clean, client=http_client),
fetch_free_dictionary_api(query_clean, client=http_client)
]
w_wiki, w_defs, d_muse, mm_data, free_dict = await asyncio.gather(*tasks)
w_wiki, w_defs, mm_data, free_dict = await asyncio.gather(*tasks)
# 1. Phonetic IPA
if free_dict.get("phonetic"):
result["phonetic"] = free_dict["phonetic"]
elif w_wiki.get("ipa"):
result["phonetic"] = w_wiki["ipa"][0]
elif d_muse.get("ipa"):
result["phonetic"] = d_muse["ipa"]
# 2. Audio URL / File (Disabled)
# 2. Audio URL / File
result["audio_file"] = ""
result["audio_url"] = ""
result["audio_url"] = free_dict.get("audio_url") or ""
# 3. Part of Speech
if w_defs.get("parts_of_speech"):
result["part_of_speech"] = ", ".join(w_defs["parts_of_speech"][:2])
elif d_muse.get("part_of_speech"):
result["part_of_speech"] = d_muse["part_of_speech"]
# 4. Definitions & Examples
if w_defs.get("definitions"):
result["definitions"] = w_defs["definitions"]
elif d_muse.get("definitions"):
result["definitions"] = d_muse["definitions"]
if w_defs.get("examples"):
result["examples"] = w_defs["examples"]