feat(reader): autonomous Wikipedia import and reading with media, tables and settings

tests/test_api.py

@@ -2046,6 +2046,191 @@ def test_clean_russian_translation_removes_romanization():
     assert clean_like == "подобный, похожий, как, типа"
 
 
+def test_extract_wikipedia_title_and_lang():
+    from app.reader import extract_wikipedia_title_and_lang
+
+    t1, l1 = extract_wikipedia_title_and_lang("https://simple.wikipedia.org/wiki/Solar_System")
+    assert t1 == "Solar_System"
+    assert l1 == "simple"
+
+    t2, l2 = extract_wikipedia_title_and_lang("https://en.wikipedia.org/wiki/Quantum_mechanics")
+    assert t2 == "Quantum_mechanics"
+    assert l2 == "en"
+
+    t3, l3 = extract_wikipedia_title_and_lang("Artificial Intelligence", default_lang="simple")
+    assert t3 == "Artificial_Intelligence"
+    assert l3 == "simple"
+
+
+def test_clean_wiki_html_unit():
+    from app.reader import clean_wiki_html
+
+    sample_parsoid_html = """
+    <!DOCTYPE html>
+    <html>
+    <body>
+        <style>.some-css { color: red; }</style>
+        <script>console.log("bad");</script>
+        <div class="hatnote">This article is about astronomy.</div>
+        <table class="infobox">
+            <tr><th>Lead</th></tr>
+            <tr><td><img src="//thumb.wikimedia.org/wikipedia/commons/thumb/a/a1/Sun.jpg/300px-Sun.jpg" alt="The Sun"></td></tr>
+            <tr><td>Orbital info</td></tr>
+        </table>
+        <p>The <b>Sun</b> is the star at the center of the Solar System<sup class="reference">[1]</sup>.</p>
+        <figure typeof="mw:File/Thumb">
+            <img src="//thumb.wikimedia.org/wikipedia/commons/thumb/b/b2/Solar_flare.jpg/400px-Solar_flare.jpg" alt="Solar flare">
+            <figcaption>A powerful <a href="./Solar_flare">solar flare</a> erupting from the surface</figcaption>
+        </figure>
+        <p>It is a nearly perfect ball of hot plasma<sup class="reference">[citation needed]</sup>.</p>
+
+        <h2><span class="mw-headline">Characteristics</span><span class="mw-editsection">[edit]</span></h2>
+        <p>The Sun is a G-type main-sequence star.</p>
+        <table class="wikitable">
+            <thead><tr><th>Parameter</th><th>Value</th></tr></thead>
+            <tbody><tr><td>Radius</td><td>696,340 km</td></tr></tbody>
+        </table>
+
+        <h2>References</h2>
+        <p>1. Solar Physics Journal 2024.</p>
+
+        <h2>Other websites</h2>
+        <ul><li>NASA Sun info</li></ul>
+    </body>
+    </html>
+    """
+
+    chapters, cover_url, total_words = clean_wiki_html(sample_parsoid_html, article_title="Sun")
+
+    assert cover_url.startswith("https://thumb.wikimedia.org/")
+    assert len(chapters) == 2  # Overview and Characteristics (References and Other websites ignored)
+
+    ch1 = chapters[0]
+    assert ch1["title"] == "Overview"
+    assert "star at the center of the Solar System" in ch1["content"]
+    assert "[1]" not in ch1["content"]
+    assert "[citation needed]" not in ch1["content"]
+    assert ".some-css" not in ch1["content"]
+    assert "bad" not in ch1["content"]
+    assert "<figure class=\"reader-figure\">" in ch1["content"]
+    assert "https://thumb.wikimedia.org/wikipedia/commons/thumb/b/b2/Solar_flare.jpg/400px-Solar_flare.jpg" in ch1["content"]
+    assert "A powerful solar flare erupting from the surface" in ch1["content"]
+
+    ch2 = chapters[1]
+    assert ch2["title"] == "Characteristics"
+    assert "[edit]" not in ch2["title"]
+    assert "G-type main-sequence star" in ch2["content"]
+    assert "reader-table" in ch2["content"]
+    assert "696,340 km" in ch2["content"]
+    assert total_words > 20
+
+
+def test_wikipedia_api_endpoints_workflow(monkeypatch):
+    import urllib.request
+    import io
+    from app.reader import clean_wiki_html
+
+    sample_html = """
+    <html><body>
+        <table class="infobox">
+            <tr><td><img src="//thumb.wikimedia.org/test/cover.jpg" alt="Planet"></td></tr>
+        </table>
+        <p>Earth is the third planet from the Sun and the only astronomical object known to harbor life.</p>
+        <h2>Atmosphere</h2>
+        <p>The atmosphere of Earth is composed mostly of nitrogen and oxygen.</p>
+    </body></html>
+    """
+
+    sample_summary = {
+        "title": "Earth",
+        "thumbnail": {"source": "https://thumb.wikimedia.org/test/cover.jpg"}
+    }
+
+    class MockResponse:
+        def __init__(self, data_bytes, status=200):
+            self.data_bytes = data_bytes
+            self.status = status
+
+        def read(self, *args, **kwargs):
+            return self.data_bytes
+
+        def decode(self, *args, **kwargs):
+            return self.data_bytes.decode(*args, **kwargs)
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            pass
+
+    def mock_urlopen(req, timeout=None):
+        url = req.full_url if hasattr(req, "full_url") else str(req)
+        if "summary" in url:
+            return MockResponse(json.dumps(sample_summary).encode("utf-8"))
+        elif "html" in url:
+            return MockResponse(sample_html.encode("utf-8"))
+        return MockResponse(b"{}")
+
+    monkeypatch.setattr(urllib.request, "urlopen", mock_urlopen)
+
+    # 1. Login user
+    login_resp = client.post("/api/login", json={"username": "testadmin", "password": "testpass12345"})
+    assert login_resp.status_code == 200
+    token = login_resp.cookies.get(config.SESSION_COOKIE_NAME)
+    auth_headers = {"Cookie": f"{config.SESSION_COOKIE_NAME}={token}"}
+
+    # 2. Fetch Wikipedia article
+    resp = client.post(
+        "/api/books/fetch-wikipedia",
+        json={"query": "Earth", "lang": "simple", "topic": "science"},
+        headers=auth_headers
+    )
+    assert resp.status_code == 200
+    book_data = resp.json()
+    assert book_data["title"] == "Earth"
+    assert book_data["source_type"] == "wikipedia"
+    assert "cover.jpg" in book_data["cover_url"]
+    assert len(book_data["chapters"]) >= 1
+    assert book_data["already_existed"] is False
+    book_id = book_data["id"]
+
+    # 3. Fetch again without force - should return already_existed=True
+    resp2 = client.post(
+        "/api/books/fetch-wikipedia",
+        json={"query": "Earth", "lang": "simple", "force": False},
+        headers=auth_headers
+    )
+    assert resp2.status_code == 200
+    assert resp2.json()["already_existed"] is True
+    assert resp2.json()["id"] == book_id
+
+    # 4. Check book list includes cover_url and source_type
+    list_resp = client.get("/api/books", headers=auth_headers)
+    assert list_resp.status_code == 200
+    books = list_resp.json()
+    wiki_book = next((b for b in books if b["id"] == book_id), None)
+    assert wiki_book is not None
+    assert wiki_book["source_type"] == "wikipedia"
+    assert "cover.jpg" in wiki_book["cover_url"]
+
+    # 5. Update reading progress
+    prog_resp = client.put(
+        f"/api/books/{book_id}/progress",
+        json={"current_chapter": 1, "current_position": 0.5},
+        headers=auth_headers
+    )
+    assert prog_resp.status_code == 200
+    assert prog_resp.json()["current_chapter"] == 1
+    assert prog_resp.json()["current_position"] == 0.5
+    assert prog_resp.json()["cover_url"] != ""
+
+    # 6. Delete book
+    del_resp = client.delete(f"/api/books/{book_id}", headers=auth_headers)
+    assert del_resp.status_code == 200
+    assert del_resp.json()["deleted_id"] == book_id
+
+
+