diff --git a/TODO.md b/TODO.md index f874aa5..8e0e494 100644 --- a/TODO.md +++ b/TODO.md @@ -175,13 +175,27 @@ --- -### 8. πŸ”Œ ΠŸΡƒΠ» соСдинСний HTTP (`httpx.AsyncClient` с Keep-Alive) -- [ ] Π’ [`app/dictionary.py`](file:///home/ai-projects/services/study.qpid.pro/app/dictionary.py) ΡΠΎΠ·Π΄Π°Ρ‚ΡŒ синглтон-ΠΊΠ»ΠΈΠ΅Π½Ρ‚ `get_http_client()` с ΠΏΡƒΠ»ΠΎΠΌ соСдинСний: +### 7.1. πŸ–ΌοΈ Π˜Π·Π²Π»Π΅Ρ‡Π΅Π½ΠΈΠ΅ ΠΈ ΠΎΡ‚ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠ΅ схСм, Ρ‚ΠΎΠΏΠΎΠ»ΠΎΠ³ΠΈΠΉ ΠΈ Π΄ΠΈΠ°Π³Ρ€Π°ΠΌΠΌ ΠΈΠ· PDF / EPUB (Planned) +- [ ] **Π˜Π·Π²Π»Π΅Ρ‡Π΅Π½ΠΈΠ΅ Π±ΠΈΠ½Π°Ρ€Π½Ρ‹Ρ… ΠΈΠ·ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠΉ Π½Π° диск**: + - ΠŸΡ€ΠΈ парсингС PDF/EPUB ΠΈΠ·Π²Π»Π΅ΠΊΠ°Ρ‚ΡŒ графичСскиС ΠΎΠ±ΡŠΠ΅ΠΊΡ‚Ρ‹ (`page.images` / Π°Ρ€Ρ…ΠΈΠ²Ρ‹ ΠΈΠ·ΠΎΠ±Ρ€Π°ΠΆΠ΅Π½ΠΈΠΉ) Π½Π° Ρ„Π°ΠΉΠ»ΠΎΠ²ΡƒΡŽ систСму Π² ΠΈΠ·ΠΎΠ»ΠΈΡ€ΠΎΠ²Π°Π½Π½ΡƒΡŽ Π΄ΠΈΡ€Π΅ΠΊΡ‚ΠΎΡ€ΠΈΡŽ ΠΊΠ½ΠΈΠ³ΠΈ: `data/books/{book_id}/images/{img_name}`. + - Π˜Π·Π±Π΅Π³Π°Ρ‚ΡŒ раздувания Π±Π°Π·Ρ‹ Π΄Π°Π½Π½Ρ‹Ρ… SQLite: Π±Π°Π·Π° Ρ…Ρ€Π°Π½ΠΈΡ‚ Ρ‚ΠΎΠ»ΡŒΠΊΠΎ тСкст ΠΈ ΠΌΠ΅Ρ‚Π°Π΄Π°Π½Π½Ρ‹Π΅, бинарная Π³Ρ€Π°Ρ„ΠΈΠΊΠ° хранится Π½Π° дискС. +- [ ] **ΠšΡΡˆΠΈΡ€ΡƒΡŽΡ‰ΠΈΠΉ ΠΌΠ΅Π΄ΠΈΠ°-эндпоинт**: + - Π‘ΠΎΠ·Π΄Π°Ρ‚ΡŒ эндпоинт `GET /api/books/{book_id}/images/{image_name}` с Π·Π°Π³ΠΎΠ»ΠΎΠ²ΠΊΠ°ΠΌΠΈ ΠΊΡΡˆΠΈΡ€ΠΎΠ²Π°Π½ΠΈΡ (`Cache-Control: public, max-age=31536000, immutable`). +- [ ] **ВстраиваниС Π² тСкст Ρ‡ΠΈΡ‚Π°Π»ΠΊΠΈ**: + - ΠŸΡ€ΠΈΠ²ΡΠ·Ρ‹Π²Π°Ρ‚ΡŒ ΠΈΠ»Π»ΡŽΡΡ‚Ρ€Π°Ρ†ΠΈΠΈ ΠΊ ΡΠΎΠΎΡ‚Π²Π΅Ρ‚ΡΡ‚Π²ΡƒΡŽΡ‰Π΅ΠΉ страницС/Π³Π»Π°Π²Π΅ Ρ‡Π΅Ρ€Π΅Π· Markdown-Ρ‚Π΅Π³ΠΈ: `![Π‘Ρ…Π΅ΠΌΠ° Ρ‚ΠΎΠΏΠΎΠ»ΠΎΠ³ΠΈΠΈ](/api/books/{book_id}/images/{img_name})`. + - Π˜Π½Ρ‚Π΅Ρ€Π°ΠΊΡ‚ΠΈΠ²Π½Ρ‹ΠΉ просмотр ΠΈ Π·ΡƒΠΌ схСм ΠΏΠΎ ΠΊΠ»ΠΈΠΊΡƒ Π² Ρ‡ΠΈΡ‚Π°Π»ΠΊΠ΅. + +--- + +### 8. πŸ”Œ ΠŸΡƒΠ» соСдинСний HTTP (`httpx.AsyncClient` с Keep-Alive) (Completed) +- [x] Π’ [`app/dictionary.py`](file:///home/ai-projects/services/study.qpid.pro/app/dictionary.py) ΡΠΎΠ·Π΄Π°Ρ‚ΡŒ синглтон-ΠΊΠ»ΠΈΠ΅Π½Ρ‚ `get_http_client()` с ΠΏΡƒΠ»ΠΎΠΌ соСдинСний: ```python - limits = httpx.Limits(max_keepalive_connections=10, max_connections=20) + limits = httpx.Limits(max_keepalive_connections=10, max_connections=20, keepalive_expiry=60.0) ``` -- [ ] Π—Π°ΠΌΠ΅Π½ΠΈΡ‚ΡŒ Π²Ρ‹Π·ΠΎΠ²Ρ‹ `async with httpx.AsyncClient()` Π² парсСрах Π½Π° ΠΏΠ΅Ρ€Π΅ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌΡ‹ΠΉ ΠΊΠ»ΠΈΠ΅Π½Ρ‚. -- [ ] Π£ΡΡ‚Ρ€Π°Π½ΠΈΡ‚ΡŒ 5 ΠΏΠΎΠ²Ρ‚ΠΎΡ€Π½Ρ‹Ρ… TCP/TLS Ρ…ΡΠ½Π΄ΡˆΠ΅ΠΉΠΊΠΎΠ² Π½Π° ΠΊΠ°ΠΆΠ΄ΠΎΠ΅ слово, сократив ΠΏΠΈΠ½Π³ Π²Π½Π΅ΡˆΠ½ΠΈΡ… запросов Π½Π° 300–600 мс. +- [x] Π—Π°ΠΌΠ΅Π½ΠΈΡ‚ΡŒ Π²Ρ‹Π·ΠΎΠ²Ρ‹ `async with httpx.AsyncClient()` Π² парсСрах Π½Π° ΠΏΠ΅Ρ€Π΅ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠ΅ΠΌΡ‹ΠΉ ΠΊΠ»ΠΈΠ΅Π½Ρ‚. +- [x] Π£ΡΡ‚Ρ€Π°Π½ΠΈΡ‚ΡŒ 5 ΠΏΠΎΠ²Ρ‚ΠΎΡ€Π½Ρ‹Ρ… TCP/TLS Ρ…ΡΠ½Π΄ΡˆΠ΅ΠΉΠΊΠΎΠ² Π½Π° ΠΊΠ°ΠΆΠ΄ΠΎΠ΅ слово, сократив Π·Π°Π΄Π΅Ρ€ΠΆΠΊΡƒ Π²Π½Π΅ΡˆΠ½ΠΈΡ… запросов Π½Π° 300–600 мс. +- [x] Π”ΠΎΠ±Π°Π²ΠΈΡ‚ΡŒ Π²Ρ‹Π·ΠΎΠ² `close_http_client()` Π² `lifespan` FastAPI ΠΏΡ€ΠΈ остановкС прилоТСния для чистого закрытия ΠΎΡ‚ΠΊΡ€Ρ‹Ρ‚Ρ‹Ρ… сокСтов. +- [x] Π”ΠΎΠ±Π°Π²ΠΈΡ‚ΡŒ ΡŽΠ½ΠΈΡ‚-тСсты `test_http_connection_pooling` Π² `tests/test_api.py` (22/22 passed). --- diff --git a/app/dictionary.py b/app/dictionary.py index 39c4161..a35f495 100644 --- a/app/dictionary.py +++ b/app/dictionary.py @@ -8,7 +8,39 @@ from app.database import get_db USER_AGENT = "NotesApp/1.0 (https://notes.qpid.pro; admin@qpid.pro)" -async def fetch_wiktionary_wikitext(word: str) -> Dict[str, Any]: +_http_client: Optional[httpx.AsyncClient] = None + + +def get_http_client() -> httpx.AsyncClient: + """ + Returns a shared singleton httpx.AsyncClient instance with connection pooling and Keep-Alive. + Reuses persistent TCP/TLS connections to dictionary providers (Wiktionary, Datamuse, MyMemory) + eliminating 300-600ms of repeated connection handshakes per word lookup. + """ + global _http_client + if _http_client is None or _http_client.is_closed: + limits = httpx.Limits(max_keepalive_connections=10, max_connections=20, keepalive_expiry=60.0) + timeout = httpx.Timeout(connect=3.0, read=4.0, write=3.0, pool=3.0) + _http_client = httpx.AsyncClient( + limits=limits, + timeout=timeout, + headers={"User-Agent": USER_AGENT}, + follow_redirects=True + ) + return _http_client + + +async def close_http_client(): + """ + Closes the shared singleton HTTP client on application shutdown. + """ + global _http_client + if _http_client is not None and not _http_client.is_closed: + await _http_client.aclose() + _http_client = None + + +async def fetch_wiktionary_wikitext(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]: url = "https://en.wiktionary.org/w/api.php" params = { "action": "parse", @@ -17,111 +49,109 @@ async def fetch_wiktionary_wikitext(word: str) -> Dict[str, Any]: "format": "json" } try: - async with httpx.AsyncClient(timeout=4.0) as client: - resp = await client.get(url, params=params, headers={"User-Agent": USER_AGENT}) - if resp.status_code == 200: - data = resp.json() - wikitext = data.get("parse", {}).get("wikitext", {}).get("*", "") - - # Extract IPA - ipa_raw = re.findall(r"\{\{IPA\|en\|([^}]+)\}\}", wikitext) - ipa_cleaned = [] - for entry in ipa_raw: - parts = entry.split("|") - for p in parts: - if p.startswith("/"): - ipa_cleaned.append(p) - - # Extract Audio file names - audio_matches = re.findall(r"\{\{audio\|en\|([^|}]+)", wikitext) - - # Extract Russian translations - ru_raw = re.findall(r"\{\{t[+-]?\|ru\|([^|}]+)", wikitext) - ru_cleaned = [] - for r in ru_raw: - # Clean wiki markup like [[word]] or stress marks - clean = r.replace("[[", "").replace("]]", "").strip() - if clean and clean not in ru_cleaned: - ru_cleaned.append(clean) + http = client or get_http_client() + resp = await http.get(url, params=params, timeout=4.0) + if resp.status_code == 200: + data = resp.json() + wikitext = data.get("parse", {}).get("wikitext", {}).get("*", "") - return { - "ipa": ipa_cleaned, - "audio": [], - "ru_translations": ru_cleaned - } + # Extract IPA + ipa_raw = re.findall(r"\{\{IPA\|en\|([^}]+)\}\}", wikitext) + ipa_cleaned = [] + for entry in ipa_raw: + parts = entry.split("|") + for p in parts: + if p.startswith("/"): + ipa_cleaned.append(p) + + # Extract Russian translations + ru_raw = re.findall(r"\{\{t[+-]?\|ru\|([^|}]+)", wikitext) + ru_cleaned = [] + for r in ru_raw: + clean = r.replace("[[", "").replace("]]", "").strip() + if clean and clean not in ru_cleaned: + ru_cleaned.append(clean) + + return { + "ipa": ipa_cleaned, + "audio": [], + "ru_translations": ru_cleaned + } except Exception: pass return {"ipa": [], "audio": [], "ru_translations": []} -async def fetch_wiktionary_definitions(word: str) -> Dict[str, Any]: + +async def fetch_wiktionary_definitions(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]: url = f"https://en.wiktionary.org/api/rest_v1/page/definition/{urllib.parse.quote(word)}" try: - async with httpx.AsyncClient(timeout=4.0) as client: - resp = await client.get(url, headers={"User-Agent": USER_AGENT}) - if resp.status_code == 200: - data = resp.json() - en_entries = data.get("en", []) - parts_of_speech = [] - definitions = [] - examples = [] - for entry in en_entries: - pos = entry.get("partOfSpeech") - if pos and pos not in parts_of_speech: - parts_of_speech.append(pos) - for d in entry.get("definitions", []): - raw_def = d.get("definition", "") - # strip html tags - clean_def = re.sub(r"<[^>]+>", "", raw_def).strip() - if clean_def and clean_def not in definitions: - definitions.append(clean_def) - for ex in d.get("examples", []): - clean_ex = re.sub(r"<[^>]+>", "", ex).strip() - if clean_ex and clean_ex not in examples: - examples.append(clean_ex) - return { - "parts_of_speech": parts_of_speech, - "definitions": definitions[:4], - "examples": examples[:3] - } + http = client or get_http_client() + resp = await http.get(url, timeout=4.0) + if resp.status_code == 200: + data = resp.json() + en_entries = data.get("en", []) + parts_of_speech = [] + definitions = [] + examples = [] + for entry in en_entries: + pos = entry.get("partOfSpeech") + if pos and pos not in parts_of_speech: + parts_of_speech.append(pos) + for d in entry.get("definitions", []): + raw_def = d.get("definition", "") + clean_def = re.sub(r"<[^>]+>", "", raw_def).strip() + if clean_def and clean_def not in definitions: + definitions.append(clean_def) + for ex in d.get("examples", []): + clean_ex = re.sub(r"<[^>]+>", "", ex).strip() + if clean_ex and clean_ex not in examples: + examples.append(clean_ex) + return { + "parts_of_speech": parts_of_speech, + "definitions": definitions[:4], + "examples": examples[:3] + } except Exception: pass return {"parts_of_speech": [], "definitions": [], "examples": []} -async def fetch_datamuse(word: str) -> Dict[str, Any]: + +async def fetch_datamuse(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]: url = "https://api.datamuse.com/words" params = {"sp": word, "md": "drp", "ipa": "1"} try: - async with httpx.AsyncClient(timeout=3.0) as client: - resp = await client.get(url, params=params) - if resp.status_code == 200: - items = resp.json() - if items and isinstance(items, list): - first = items[0] - tags = first.get("tags", []) - ipa = None - pos = None - for t in tags: - if t.startswith("ipa_pron:"): - ipa = "/" + t.split(":", 1)[1] + "/" - elif t in ("n", "v", "adj", "adv"): - pos_map = {"n": "Noun", "v": "Verb", "adj": "Adjective", "adv": "Adverb"} - pos = pos_map.get(t) - - defs = [] - for d in first.get("defs", []): - parts = d.split("\t", 1) - defs.append(parts[1].strip() if len(parts) > 1 else parts[0].strip()) + http = client or get_http_client() + resp = await http.get(url, params=params, timeout=3.0) + if resp.status_code == 200: + items = resp.json() + if items and isinstance(items, list): + first = items[0] + tags = first.get("tags", []) + ipa = None + pos = None + for t in tags: + if t.startswith("ipa_pron:"): + ipa = "/" + t.split(":", 1)[1] + "/" + elif t in ("n", "v", "adj", "adv"): + pos_map = {"n": "Noun", "v": "Verb", "adj": "Adjective", "adv": "Adverb"} + pos = pos_map.get(t) - return { - "ipa": ipa, - "part_of_speech": pos, - "definitions": defs[:3] - } + defs = [] + for d in first.get("defs", []): + parts = d.split("\t", 1) + defs.append(parts[1].strip() if len(parts) > 1 else parts[0].strip()) + + return { + "ipa": ipa, + "part_of_speech": pos, + "definitions": defs[:3] + } except Exception: pass return {"ipa": None, "part_of_speech": None, "definitions": []} -async def fetch_mymemory(query: str) -> Dict[str, Any]: + +async def fetch_mymemory(query: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]: url = "https://api.mymemory.translated.net/get" params = { "q": query, @@ -129,45 +159,46 @@ async def fetch_mymemory(query: str) -> Dict[str, Any]: "de": config.MYMEMORY_EMAIL } try: - async with httpx.AsyncClient(timeout=4.0) as client: - resp = await client.get(url, params=params) - if resp.status_code == 200: - data = resp.json() - main_trans = data.get("responseData", {}).get("translatedText", "").strip() - matches = data.get("matches", []) - alt_list = [] - if main_trans: - alt_list.append(main_trans) - for m in matches: - t = m.get("translation", "").strip() - if t and t not in alt_list and t.lower() != query.lower(): - alt_list.append(t) - return { - "translation": main_trans, - "matches": alt_list[:6] - } + http = client or get_http_client() + resp = await http.get(url, params=params, timeout=4.0) + if resp.status_code == 200: + data = resp.json() + main_trans = data.get("responseData", {}).get("translatedText", "").strip() + matches = data.get("matches", []) + alt_list = [] + if main_trans: + alt_list.append(main_trans) + for m in matches: + t = m.get("translation", "").strip() + if t and t not in alt_list and t.lower() != query.lower(): + alt_list.append(t) + return { + "translation": main_trans, + "matches": alt_list[:6] + } except Exception: pass return {"translation": "", "matches": []} -async def fetch_free_dictionary_api(word: str) -> Dict[str, Any]: + +async def fetch_free_dictionary_api(word: str, client: Optional[httpx.AsyncClient] = None) -> Dict[str, Any]: """Optional tier: api.dictionaryapi.dev with fast timeout.""" url = f"https://api.dictionaryapi.dev/api/v2/entries/en/{urllib.parse.quote(word)}" try: - async with httpx.AsyncClient(timeout=1.5) as client: - resp = await client.get(url) - if resp.status_code == 200: - data = resp.json()[0] - phonetic = data.get("phonetic") - audio_url = None - for p in data.get("phonetics", []): - if p.get("audio"): - audio_url = p.get("audio") - break - return { - "phonetic": phonetic, - "audio_url": audio_url - } + http = client or get_http_client() + resp = await http.get(url, timeout=1.5) + if resp.status_code == 200: + data = resp.json()[0] + phonetic = data.get("phonetic") + audio_url = None + for p in data.get("phonetics", []): + if p.get("audio"): + audio_url = p.get("audio") + break + return { + "phonetic": phonetic, + "audio_url": audio_url + } except Exception: pass return {"phonetic": None, "audio_url": None} @@ -208,19 +239,21 @@ async def lookup_term(query: str) -> Dict[str, Any]: # Parallel queries via asyncio import asyncio + http_client = get_http_client() + if is_sentence: # For long sentences, primary focus is translation - mm_data = await fetch_mymemory(query_clean) + mm_data = await fetch_mymemory(query_clean, client=http_client) result["translation"] = mm_data.get("translation", "") result["translations_list"] = mm_data.get("matches", []) else: - # For words and short phrases + # For words and short phrases: reuse keep-alive connections across providers tasks = [ - fetch_wiktionary_wikitext(query_clean), - fetch_wiktionary_definitions(query_clean), - fetch_datamuse(query_clean), - fetch_mymemory(query_clean), - fetch_free_dictionary_api(query_clean) + fetch_wiktionary_wikitext(query_clean, client=http_client), + fetch_wiktionary_definitions(query_clean, client=http_client), + fetch_datamuse(query_clean, client=http_client), + fetch_mymemory(query_clean, client=http_client), + fetch_free_dictionary_api(query_clean, client=http_client) ] w_wiki, w_defs, d_muse, mm_data, free_dict = await asyncio.gather(*tasks) diff --git a/app/main.py b/app/main.py index edee002..aca7a56 100644 --- a/app/main.py +++ b/app/main.py @@ -25,7 +25,7 @@ from app.auth import ( record_login_success, change_user_password, ) -from app.dictionary import lookup_term, USER_AGENT +from app.dictionary import lookup_term, USER_AGENT, close_http_client from app.export import export_csv, export_json, export_anki_tsv, export_markdown from app.srs import calculate_sm2, generate_quiz_options, build_activity_heatmap, calculate_cefr_level from app.reader import parse_epub, parse_text, parse_pdf, count_words @@ -36,6 +36,7 @@ async def lifespan(app: FastAPI): init_db() ensure_admin_user() yield + await close_http_client() app = FastAPI( diff --git a/tests/test_api.py b/tests/test_api.py index 04958a8..4293383 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1011,3 +1011,34 @@ def test_books_pdf_multipart_upload(): # Cleanup client.delete(f"/api/books/{book_id}") + + +def test_http_connection_pooling(): + import pytest + from app.dictionary import get_http_client, close_http_client, lookup_term + import asyncio + + # 1. Singleton returns same instance + c1 = get_http_client() + c2 = get_http_client() + assert c1 is c2 + assert not c1.is_closed + + # 2. Check connection limits configuration + pool = c1._transport._pool + assert pool._max_keepalive_connections == 10 + assert pool._max_connections == 20 + + # 3. Lookup using pooled client + res = asyncio.run(lookup_term("resilience")) + assert res["query"] == "resilience" + assert "translation" in res + + # 4. Clean closure on shutdown + asyncio.run(close_http_client()) + assert c1.is_closed + # Next call creates a new active client + c3 = get_http_client() + assert c3 is not c1 + assert not c3.is_closed + asyncio.run(close_http_client())