feat(reader): autonomous Wikipedia import and reading with media, tables and settings
This commit is contained in:
+381
-1
@@ -11,7 +11,7 @@ import urllib.error
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
from typing import Tuple, List, Dict
|
||||
from typing import Tuple, List, Dict, Optional
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
@@ -1087,3 +1087,383 @@ def fetch_and_parse_rfc(query: str) -> Tuple[str, str, List[Dict[str, str]], int
|
||||
return clean_title, clean_author, chapters, rfc_num, total_words
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wikipedia Autonomous Import & Reader Module
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WIKIPEDIA_USER_AGENT = "StudyReaderApp/1.0 (https://study.qpid.pro; admin@study.qpid.pro) edu-reader"
|
||||
|
||||
WIKIPEDIA_TOPIC_POOLS: Dict[str, Dict[str, List[str]]] = {
|
||||
"science": {
|
||||
"simple": [
|
||||
"Solar_System", "Sun", "Moon", "Mars", "Black_hole", "Milky_Way", "Speed_of_light",
|
||||
"Gravity", "Atom", "Dinosaur", "Evolution", "Photosynthesis", "Asteroid", "Comet",
|
||||
"Volcano", "Plate_tectonics", "Hubble_Space_Telescope", "James_Webb_Space_Telescope",
|
||||
"Planet", "Universe", "Earthquake", "Atmosphere"
|
||||
],
|
||||
"en": [
|
||||
"Solar_System", "Sun", "Moon", "Mars", "Black_hole", "Milky_Way", "General_relativity",
|
||||
"Quantum_mechanics", "Speed_of_light", "Plate_tectonics", "Supernova", "Exoplanet",
|
||||
"Dark_matter", "Hubble_Space_Telescope", "James_Webb_Space_Telescope", "Pendulum",
|
||||
"Thermodynamics", "Stellar_evolution", "Gravitational_wave"
|
||||
]
|
||||
},
|
||||
"tech": {
|
||||
"simple": [
|
||||
"Computer", "Internet", "World_Wide_Web", "Artificial_intelligence", "Smartphone",
|
||||
"Robot", "Video_game", "Software", "Linux", "Algorithm", "Satellite", "Electricity",
|
||||
"Machine_learning", "Computer_network", "Fiber_optic", "Microprocessor"
|
||||
],
|
||||
"en": [
|
||||
"Artificial_intelligence", "Computer", "Internet", "World_Wide_Web", "Quantum_computing",
|
||||
"Machine_learning", "Operating_system", "Cryptography", "Blockchain", "Transistor",
|
||||
"Linux", "Neural_network", "Optical_fiber", "Semiconductor"
|
||||
]
|
||||
},
|
||||
"history": {
|
||||
"simple": [
|
||||
"Ancient_Egypt", "Roman_Empire", "Ancient_Greece", "Middle_Ages", "Industrial_Revolution",
|
||||
"Silk_Road", "Renaissance", "Aztec", "World_War_I", "French_Revolution", "Great_Wall_of_China",
|
||||
"Mesopotamia", "Byzantine_Empire", "Inca_Empire", "Voyages_of_Christopher_Columbus"
|
||||
],
|
||||
"en": [
|
||||
"Ancient_Egypt", "Roman_Empire", "Renaissance", "Industrial_Revolution", "Silk_Road",
|
||||
"Byzantine_Empire", "Age_of_Discovery", "Classical_Athens", "Mongol_Empire", "French_Revolution",
|
||||
"Machu_Picchu", "Hanseatic_League", "Printing_press", "Maya_civilization"
|
||||
]
|
||||
},
|
||||
"nature": {
|
||||
"simple": [
|
||||
"Blue_whale", "Tiger", "African_elephant", "Honey_bee", "Amazon_rainforest", "Coral_reef",
|
||||
"Penguin", "Dolphin", "Wolf", "Sahara", "Giant_panda", "Cheetah", "Kangaroo", "Giraffe",
|
||||
"Ocean", "Forest", "Rainforest", "Migration"
|
||||
],
|
||||
"en": [
|
||||
"Blue_whale", "Tiger", "African_elephant", "Amazon_rainforest", "Great_Barrier_Reef",
|
||||
"Deep_sea", "Polar_bear", "Galapagos_Islands", "Bioluminescence", "Coast_redwood",
|
||||
"Honey_bee", "Monarch_butterfly", "Humpback_whale", "Ecosystem"
|
||||
]
|
||||
},
|
||||
"psychology": {
|
||||
"simple": [
|
||||
"Memory", "Emotion", "Consciousness", "Sleep", "Brain", "Intelligence", "Dream",
|
||||
"Perception", "Learning", "Happiness", "Stress_(biology)", "Mind", "Behavior"
|
||||
],
|
||||
"en": [
|
||||
"Human_memory", "Emotion", "Consciousness", "Sleep", "Cognitive_bias", "Neuroplasticity",
|
||||
"Empathy", "Perception", "Motivation", "Decision-making", "Working_memory",
|
||||
"Attention", "Lateralization_of_brain_function"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def extract_wikipedia_title_and_lang(query: str, default_lang: str = "simple") -> Tuple[str, str]:
|
||||
"""
|
||||
Parses a Wikipedia URL or plain title into (article_title, lang).
|
||||
Examples:
|
||||
https://simple.wikipedia.org/wiki/Solar_System -> ('Solar_System', 'simple')
|
||||
https://en.wikipedia.org/wiki/Quantum_mechanics -> ('Quantum_mechanics', 'en')
|
||||
'Quantum mechanics' -> ('Quantum_mechanics', default_lang)
|
||||
"""
|
||||
raw = (query or "").strip()
|
||||
if not raw:
|
||||
return "", default_lang
|
||||
|
||||
# Check URL format
|
||||
m_url = re.search(r"https?://([a-z]+)\.(?:m\.)?wikipedia\.org/wiki/([^?#]+)", raw, re.IGNORECASE)
|
||||
if m_url:
|
||||
lang = m_url.group(1).lower()
|
||||
title_part = urllib.parse.unquote(m_url.group(2)).strip()
|
||||
# Convert spaces
|
||||
clean_title = title_part.replace(" ", "_")
|
||||
return clean_title, lang
|
||||
|
||||
# Plain title or search query
|
||||
clean_title = raw.replace(" ", "_")
|
||||
return clean_title, default_lang
|
||||
|
||||
|
||||
def clean_wiki_html(raw_html: str, article_title: str = "") -> Tuple[List[Dict[str, str]], str, int]:
|
||||
"""
|
||||
Parses MediaWiki Parsoid HTML into clean, structured chapters for the interactive reader.
|
||||
Extracts cover image/thumbnail, preserves figures, animated GIFs/SVGs, and clean data tables,
|
||||
and strips wiki clutter ([1], [citation needed], infobox rows, navboxes).
|
||||
"""
|
||||
cover_image = ""
|
||||
|
||||
# 1. Lead image: check infobox or first figure
|
||||
lead_img_m = re.search(
|
||||
r'<table\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?<img\b([^>]*)>',
|
||||
raw_html,
|
||||
re.DOTALL | re.IGNORECASE
|
||||
)
|
||||
if not lead_img_m:
|
||||
lead_img_m = re.search(r'<figure\b[^>]*>.*?<img\b([^>]*)>', raw_html, re.DOTALL | re.IGNORECASE)
|
||||
|
||||
if lead_img_m:
|
||||
src_m = re.search(r'src=["\x27]([^\s"\x27]+)["\x27]', lead_img_m.group(1), re.IGNORECASE)
|
||||
if src_m:
|
||||
src = src_m.group(1)
|
||||
if src.startswith("//"):
|
||||
src = "https:" + src
|
||||
elif not src.startswith("http"):
|
||||
src = "https://" + src.lstrip("/")
|
||||
cover_image = src
|
||||
|
||||
# 2. Extract body
|
||||
body_m = re.search(r"<body[^>]*>(.*?)</body>", raw_html, re.DOTALL | re.IGNORECASE)
|
||||
body = body_m.group(1) if body_m else raw_html
|
||||
|
||||
# Strip style, script, noscript tags
|
||||
body = re.sub(r'<style\b[^>]*>.*?</style>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<script\b[^>]*>.*?</script>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<noscript\b[^>]*>.*?</noscript>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 3. Strip unwanted wiki elements
|
||||
# Remove references: <sup class="reference">...</sup> and bracketed reference markers
|
||||
body = re.sub(r'<sup\b[^>]*class=["\x27][^"\x27]*reference[^"\x27]*["\x27][^>]*>.*?</sup>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<sup\b[^>]*>\s*\[\s*[\d\w\s,-]+\s*\]\s*</sup>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<span\b[^>]*class=["\x27][^"\x27]*mw-editsection[^"\x27]*["\x27][^>]*>.*?</span>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<div\b[^>]*class=["\x27][^"\x27]*(?:navbox|ambox|metadata|hatnote|sistersitebox|portal)[^"\x27]*["\x27][^>]*>.*?</div>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<table\b[^>]*class=["\x27][^"\x27]*(?:navbox|ambox|metadata|nowraplinks)[^"\x27]*["\x27][^>]*>.*?</table>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Remove infobox entirely from text flow (lead image was already captured)
|
||||
body = re.sub(r'<table\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?</table>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
body = re.sub(r'<div\b[^>]*class=["\x27][^"\x27]*infobox[^"\x27]*["\x27][^>]*>.*?</div>', '', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 4. Standardize figures and images (preserving animated GIFs/SVGs/PNGs)
|
||||
def clean_figure(m: re.Match) -> str:
|
||||
f_html = m.group(0)
|
||||
img_m = re.search(r'<img\b([^>]*)>', f_html, re.IGNORECASE)
|
||||
if not img_m:
|
||||
return ""
|
||||
attrs = img_m.group(1)
|
||||
src_m = re.search(r'src=["\x27]([^\s"\x27]+)["\x27]', attrs, re.IGNORECASE)
|
||||
if not src_m:
|
||||
return ""
|
||||
src = src_m.group(1)
|
||||
if src.startswith("//"):
|
||||
src = "https:" + src
|
||||
elif not src.startswith("http"):
|
||||
src = "https://" + src.lstrip("/")
|
||||
|
||||
alt_m = re.search(r'alt=["\x27]([^\"]*)["\x27]', attrs, re.IGNORECASE)
|
||||
alt = alt_m.group(1) if alt_m else ""
|
||||
|
||||
cap_m = re.search(r'<figcaption\b[^>]*>(.*?)</figcaption>', f_html, re.DOTALL | re.IGNORECASE)
|
||||
cap = ""
|
||||
if cap_m:
|
||||
cap_clean = re.sub(r'<[^>]+>', ' ', cap_m.group(1))
|
||||
cap = " ".join(cap_clean.split()).strip()
|
||||
|
||||
return f'\n\n<figure class="reader-figure"><img src="{src}" alt="{html.escape(alt)}" loading="lazy" /><figcaption>{html.escape(cap)}</figcaption></figure>\n\n'
|
||||
|
||||
body = re.sub(r'<figure\b[^>]*>.*?</figure>', clean_figure, body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 5. Clean data tables (tables that are not navboxes/infoboxes)
|
||||
def clean_data_table(m: re.Match) -> str:
|
||||
tbl_html = m.group(0)
|
||||
if re.search(r'class=["\x27][^"\x27]*(?:navbox|ambox|metadata|nowraplinks|infobox)[^"\x27]*["\x27]', tbl_html, re.IGNORECASE):
|
||||
return ""
|
||||
cleaned_tbl = re.sub(
|
||||
r'\s+(?:style|bgcolor|align|valign|width|height|border|cellspacing|cellpadding|id)=["\x27][^"\x27]*["\x27]',
|
||||
'',
|
||||
tbl_html,
|
||||
flags=re.IGNORECASE
|
||||
)
|
||||
cleaned_tbl = re.sub(r'<table[^>]*>', '<div class="reader-table-wrap"><table class="reader-table">', cleaned_tbl, count=1, flags=re.IGNORECASE)
|
||||
cleaned_tbl = re.sub(r'</table>', '</table></div>', cleaned_tbl, count=1, flags=re.IGNORECASE)
|
||||
return f'\n\n{cleaned_tbl}\n\n'
|
||||
|
||||
body = re.sub(r'<table\b[^>]*>.*?</table>', clean_data_table, body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# 6. Split into sections by <h2>
|
||||
ignore_sections = {
|
||||
"references", "reference", "see also", "external links", "further reading",
|
||||
"notes", "footnotes", "sources", "bibliography", "other websites",
|
||||
"related pages", "citations", "works cited", "navigation menu"
|
||||
}
|
||||
|
||||
parts = re.split(r'(<h2\b[^>]*>.*?</h2>)', body, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
raw_chapters: List[Tuple[str, str]] = []
|
||||
lead_content = parts[0].strip()
|
||||
if lead_content:
|
||||
raw_chapters.append(("Overview", lead_content))
|
||||
|
||||
idx = 1
|
||||
while idx < len(parts):
|
||||
h2_tag = parts[idx]
|
||||
h_title = re.sub(r'<[^>]+>', '', h2_tag).strip()
|
||||
h_title_clean = " ".join(h_title.split())
|
||||
sec_content = parts[idx + 1].strip() if idx + 1 < len(parts) else ""
|
||||
idx += 2
|
||||
|
||||
if h_title_clean.lower() in ignore_sections:
|
||||
continue
|
||||
|
||||
raw_chapters.append((h_title_clean, sec_content))
|
||||
|
||||
# 7. Convert HTML within each chapter into clean reader content
|
||||
chapters: List[Dict[str, str]] = []
|
||||
|
||||
def format_section_body(sec_html: str) -> str:
|
||||
s = re.sub(r'<h3\b[^>]*>(.*?)</h3>', r'\n\n### \1\n\n', sec_html, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<h4\b[^>]*>(.*?)</h4>', r'\n\n#### \1\n\n', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<li\b[^>]*>(.*?)</li>', r'\n- \1', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'</?[uo]l[^>]*>', '\n', s, flags=re.IGNORECASE)
|
||||
s = re.sub(r'<p\b[^>]*>(.*?)</p>', r'\n\n\1\n\n', s, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Protect <figure...> and <div class="reader-table-wrap">
|
||||
blocks_protected = []
|
||||
def stash_block(match: re.Match) -> str:
|
||||
token = f"§§BLOCK_{len(blocks_protected)}§§"
|
||||
blocks_protected.append(match.group(0))
|
||||
return f"\n\n{token}\n\n"
|
||||
|
||||
s = re.sub(r'<figure\b[^>]*>.*?</figure>', stash_block, s, flags=re.DOTALL | re.IGNORECASE)
|
||||
s = re.sub(r'<div class="reader-table-wrap">.*?</div>', stash_block, s, flags=re.DOTALL | re.IGNORECASE)
|
||||
|
||||
s = re.sub(r'<[^>]+>', ' ', s)
|
||||
s = html.unescape(s)
|
||||
|
||||
lines = []
|
||||
for line in s.splitlines():
|
||||
trimmed = " ".join(line.split())
|
||||
if trimmed:
|
||||
lines.append(trimmed)
|
||||
elif lines and lines[-1] != "":
|
||||
lines.append("")
|
||||
|
||||
res_text = "\n".join(lines).strip()
|
||||
|
||||
for b_idx, block_html in enumerate(blocks_protected):
|
||||
res_text = res_text.replace(f"§§BLOCK_{b_idx}§§", f"\n\n{block_html}\n\n")
|
||||
|
||||
res_text = re.sub(r'\n{3,}', '\n\n', res_text).strip()
|
||||
return res_text
|
||||
|
||||
for title, content_html in raw_chapters:
|
||||
cleaned_body = format_section_body(content_html)
|
||||
w_count = count_words(cleaned_body)
|
||||
if w_count >= 15 or "<figure" in cleaned_body or "<table" in cleaned_body:
|
||||
chapters.append({
|
||||
"title": title,
|
||||
"content": cleaned_body
|
||||
})
|
||||
|
||||
if not chapters and raw_chapters:
|
||||
for title, content_html in raw_chapters:
|
||||
cleaned_body = format_section_body(content_html)
|
||||
if cleaned_body.strip():
|
||||
chapters.append({"title": title, "content": cleaned_body.strip()})
|
||||
|
||||
total_words = sum(count_words(c["content"]) for c in chapters)
|
||||
return chapters, cover_image, total_words
|
||||
|
||||
|
||||
def fetch_and_parse_wikipedia(
|
||||
title_or_url: Optional[str] = None,
|
||||
lang: str = "simple",
|
||||
time_budget: str = "10-15",
|
||||
topic: str = "random",
|
||||
) -> Tuple[str, str, List[Dict[str, str]], str, int, str]:
|
||||
"""
|
||||
Fetches a Wikipedia article (by title/URL or by random topic & time budget),
|
||||
parses into chapters with preserved media and tables, and returns:
|
||||
(clean_title, author, chapters, cover_url, total_words, wiki_url)
|
||||
"""
|
||||
import random
|
||||
from datetime import datetime
|
||||
|
||||
clean_lang = "en" if (lang or "").lower() in {"en", "standard", "standard english"} else "simple"
|
||||
target_title = ""
|
||||
|
||||
# Case 1: Specific title or URL provided
|
||||
if title_or_url and title_or_url.strip():
|
||||
extracted_title, detected_lang = extract_wikipedia_title_and_lang(title_or_url, default_lang=clean_lang)
|
||||
target_title = extracted_title
|
||||
if detected_lang:
|
||||
clean_lang = detected_lang
|
||||
|
||||
# Case 2: Pick random article by topic and time budget
|
||||
if not target_title:
|
||||
candidates = []
|
||||
clean_topic = (topic or "random").lower().strip()
|
||||
|
||||
# Handle 'featured' article of the day
|
||||
if clean_topic == "featured":
|
||||
try:
|
||||
now = datetime.utcnow()
|
||||
feat_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/feed/featured/{now.year}/{now.month:02d}/{now.day:02d}"
|
||||
req = urllib.request.Request(feat_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=5) as r:
|
||||
feed_data = json.loads(r.read().decode("utf-8"))
|
||||
if "tfa" in feed_data and feed_data["tfa"].get("title"):
|
||||
target_title = feed_data["tfa"]["title"]
|
||||
elif "mostread" in feed_data and feed_data["mostread"].get("articles"):
|
||||
art_list = [a["title"] for a in feed_data["mostread"]["articles"] if a.get("title")]
|
||||
if art_list:
|
||||
target_title = random.choice(art_list[:5])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not target_title:
|
||||
if clean_topic in WIKIPEDIA_TOPIC_POOLS:
|
||||
candidates = list(WIKIPEDIA_TOPIC_POOLS[clean_topic].get(clean_lang, []))
|
||||
else:
|
||||
for pool_dict in WIKIPEDIA_TOPIC_POOLS.values():
|
||||
candidates.extend(pool_dict.get(clean_lang, []))
|
||||
|
||||
random.shuffle(candidates)
|
||||
target_title = candidates[0] if candidates else "Solar_System"
|
||||
|
||||
# Fetch article summary to get clean title and thumbnail
|
||||
clean_title = target_title.replace("_", " ")
|
||||
cover_url = ""
|
||||
wiki_url = f"https://{clean_lang}.wikipedia.org/wiki/{urllib.parse.quote(target_title)}"
|
||||
|
||||
try:
|
||||
sum_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/page/summary/{urllib.parse.quote(target_title)}"
|
||||
req = urllib.request.Request(sum_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
sum_data = json.loads(r.read().decode("utf-8"))
|
||||
if sum_data.get("title"):
|
||||
clean_title = sum_data["title"]
|
||||
if sum_data.get("thumbnail") and sum_data["thumbnail"].get("source"):
|
||||
cover_url = sum_data["thumbnail"]["source"]
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
raise ValueError(f"Статья «{clean_title}» не найдена в Википедии ({clean_lang}.wikipedia.org)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fetch full Parsoid HTML
|
||||
raw_html = ""
|
||||
try:
|
||||
html_url = f"https://{clean_lang}.wikipedia.org/api/rest_v1/page/html/{urllib.parse.quote(target_title)}"
|
||||
req = urllib.request.Request(html_url, headers={"User-Agent": WIKIPEDIA_USER_AGENT})
|
||||
with urllib.request.urlopen(req, timeout=12) as r:
|
||||
raw_html = r.read().decode("utf-8", errors="replace")
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
raise ValueError(f"Статья «{clean_title}» не найдена в Википедии ({clean_lang}.wikipedia.org)")
|
||||
raise ValueError(f"Ошибка загрузки статьи из Wikipedia: HTTP {he.code}")
|
||||
except Exception as e:
|
||||
raise ValueError(f"Не удалось соединиться с сервером Википедии: {e}")
|
||||
|
||||
chapters, html_cover, total_words = clean_wiki_html(raw_html, article_title=clean_title)
|
||||
if not cover_url and html_cover:
|
||||
cover_url = html_cover
|
||||
|
||||
if not chapters:
|
||||
raise ValueError(f"В статье «{clean_title}» не найдено текстового содержимого для чтения")
|
||||
|
||||
edition_name = "Simple English" if clean_lang == "simple" else "Standard English"
|
||||
author = f"Wikipedia ({edition_name})"
|
||||
|
||||
return clean_title, author, chapters, cover_url, total_words, wiki_url
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user