feat(reader): добавить парсер RFC из RFC Editor и отдельную группу спецификаций на книжной полке

- Реализован парсер RFC в app/reader.py с авто-загрузкой с rfc-editor.org и datatracker.ietf.org
- Очистка пагинации ([Page X], \x0c) и разбиение спецификаций на структурированные главы (TOC, разделы)
- Эндпоинт POST /api/books/fetch-rfc с защитой от дублирования и поддержкой номеров/URL
- Визуальное разделение на книжной полке через hr-разделитель и блок спецификаций IETF & RFC
- Быстрые пресеты стандартов (HTTP/1.1, HTTP/2, TLS 1.3, OAuth 2.0, WebSocket, QUIC, TCP, DNS)
- Интеграция в интерактивную читалку (перевод по клику, авто-захват контекста, SRS карточки)
- Unit-тесты парсера и API (31/31 passed) и обновление Service Worker кэша
This commit is contained in:
2026-10-06 21:46:07 +03:00
parent e73d3470b3
commit 82b7a654ca
8 changed files with 877 additions and 66 deletions
+112
View File
@@ -1645,6 +1645,118 @@ def test_background_tasks_and_book_mining():
assert cancel_resp.json()["cancelled"] is True
def test_rfc_parser_and_api(monkeypatch):
from app.reader import extract_rfc_number, parse_rfc_text
# 1. Number extraction
assert extract_rfc_number("9110") == 9110
assert extract_rfc_number("RFC 8446") == 8446
assert extract_rfc_number("rfc-6749") == 6749
assert extract_rfc_number("https://www.rfc-editor.org/rfc/rfc7540.txt") == 7540
assert extract_rfc_number("https://datatracker.ietf.org/doc/rfc9000/") == 9000
with pytest.raises(ValueError):
extract_rfc_number("invalid-rfc-string-without-number")
with pytest.raises(ValueError):
extract_rfc_number("rfc -5")
# 2. Parsing raw text with pagination artifacts
sample_rfc = """Internet Engineering Task Force (IETF)
Request for Comments: 9999
Category: Standards Track
Test Protocol Specification
Abstract
This is a test abstract for RFC 9999 demonstrating cleaning.
Table of Contents
1. Introduction .................................................... 1
2. Protocol Framing ................................................ 2
Authors' Addresses ................................................. 3
Standards Track [Page 1]
\x0c
RFC 9999 Test Protocol Specification June 2026
1. Introduction
The Test Protocol is designed for testing RFC parsers.
It provides ultra-clean chapter segmentation.
2. Protocol Framing
Framing uses standard binary encodings.
Authors' Addresses
Jane Doe
Example Corp
"""
clean_title, author, chapters, total_words = parse_rfc_text(sample_rfc, rfc_number=9999)
assert clean_title == "RFC 9999: Test Protocol Specification"
assert author == "IETF"
assert len(chapters) >= 3
# Check that pagination footer/header was stripped
for ch in chapters:
assert "[Page 1]" not in ch["content"]
assert "\x0c" not in ch["content"]
assert any("Introduction" in ch["title"] for ch in chapters)
assert any("Framing" in ch["title"] for ch in chapters)
assert any("Addresses" in ch["title"] for ch in chapters)
assert total_words > 20
# 3. Test API endpoint unauthenticated
anon_client = TestClient(app)
resp = anon_client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
assert resp.status_code == 401
# 4. Test API endpoint authenticated with mock fetch
import app.main as main_mod
def mock_fetch_and_parse_rfc(query):
return ("RFC 9999: Test Protocol Specification", "IETF (Jane Doe)", chapters, 9999, total_words)
monkeypatch.setattr(main_mod, "fetch_and_parse_rfc", mock_fetch_and_parse_rfc)
# Login demo user
login_resp = client.post("/api/login", json={"username": "demo_user", "password": "demo_pass_999"})
assert login_resp.status_code == 200
# Invalid input
bad_resp = client.post("/api/books/fetch-rfc", json={"rfc": "no_number_here"})
assert bad_resp.status_code == 400
# Fetch valid RFC
fetch_resp = client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
assert fetch_resp.status_code == 200
book_data = fetch_resp.json()
assert book_data["source_type"] == "rfc"
assert book_data["title"] == "RFC 9999: Test Protocol Specification"
assert book_data["already_existed"] is False
assert len(book_data["chapters"]) >= 3
book_id = book_data["id"]
# Fetch again without force -> should return existing book
fetch_again = client.post("/api/books/fetch-rfc", json={"rfc": "9999"})
assert fetch_again.status_code == 200
again_data = fetch_again.json()
assert again_data["id"] == book_id
assert again_data["already_existed"] is True
# Check bookshelf retrieval includes RFC book
shelf_resp = client.get("/api/books")
assert shelf_resp.status_code == 200
shelf_books = shelf_resp.json()
rfc_books = [b for b in shelf_books if b["source_type"] == "rfc"]
assert len(rfc_books) > 0
assert rfc_books[0]["id"] == book_id