feat: implement Stage 5 (Interactive Reader, EPUB/TXT parser, Bookshelf, Click-to-Translate) and Stage 4.3 (Strict CEFR level calculation)
This commit is contained in:
+350
@@ -0,0 +1,350 @@
|
||||
import io
|
||||
import re
|
||||
import html
|
||||
import zipfile
|
||||
import posixpath
|
||||
import urllib.parse
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
from typing import Tuple, List, Dict
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
"""
|
||||
Counts words in text across multiple languages, ignoring isolated punctuation.
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
return len(re.findall(r"\b[\w]+(?:['’][\w]+)?\b", text))
|
||||
|
||||
|
||||
class HTMLTextExtractor(HTMLParser):
|
||||
"""
|
||||
Strips HTML tags cleanly while preserving paragraph and heading breaks with newlines.
|
||||
Also extracts the first heading and/or <title> tag for chapter naming.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.text_parts: List[str] = []
|
||||
self.ignore_tags = {"style", "script", "noscript"}
|
||||
self.ignore_depth = 0
|
||||
self.heading: str | None = None
|
||||
self.in_heading = False
|
||||
self.heading_parts: List[str] = []
|
||||
self.title: str | None = None
|
||||
self.in_title = False
|
||||
self.title_parts: List[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs):
|
||||
t = tag.lower()
|
||||
if t in self.ignore_tags:
|
||||
self.ignore_depth += 1
|
||||
elif t == "title":
|
||||
self.in_title = True
|
||||
self.title_parts = []
|
||||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||
self.text_parts.append("\n\n")
|
||||
if not self.heading:
|
||||
self.in_heading = True
|
||||
self.heading_parts = []
|
||||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||||
self.text_parts.append("\n\n")
|
||||
elif t in {"br", "hr", "li", "tr"}:
|
||||
self.text_parts.append("\n")
|
||||
|
||||
def handle_endtag(self, tag: str):
|
||||
t = tag.lower()
|
||||
if t in self.ignore_tags:
|
||||
self.ignore_depth = max(0, self.ignore_depth - 1)
|
||||
elif t == "title":
|
||||
self.in_title = False
|
||||
self.title = "".join(self.title_parts).strip()
|
||||
elif t in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||
self.text_parts.append("\n\n")
|
||||
if self.in_heading:
|
||||
self.in_heading = False
|
||||
if not self.heading:
|
||||
self.heading = "".join(self.heading_parts).strip()
|
||||
elif t in {"p", "div", "section", "article", "blockquote", "header", "footer"}:
|
||||
self.text_parts.append("\n\n")
|
||||
elif t in {"li", "tr"}:
|
||||
self.text_parts.append("\n")
|
||||
|
||||
def handle_data(self, data: str):
|
||||
if self.in_title:
|
||||
self.title_parts.append(data)
|
||||
if self.ignore_depth == 0 and not self.in_title:
|
||||
# Replace non-breaking spaces with standard space
|
||||
cleaned_data = data.replace("\xa0", " ")
|
||||
self.text_parts.append(cleaned_data)
|
||||
if self.in_heading:
|
||||
self.heading_parts.append(cleaned_data)
|
||||
|
||||
def get_text(self) -> str:
|
||||
raw = "".join(self.text_parts)
|
||||
lines = [line.strip() for line in raw.split("\n")]
|
||||
clean = "\n".join(lines)
|
||||
return re.sub(r"\n{3,}", "\n\n", clean).strip()
|
||||
|
||||
|
||||
def parse_epub(file_bytes: bytes) -> Tuple[str, str, List[Dict[str, str]]]:
|
||||
"""
|
||||
Parses an EPUB file archive and extracts:
|
||||
- title: book title
|
||||
- author: creator/author
|
||||
- chapters: list of {"title": str, "content": str}
|
||||
"""
|
||||
try:
|
||||
z = zipfile.ZipFile(io.BytesIO(file_bytes))
|
||||
except Exception as e:
|
||||
raise ValueError(f"Invalid EPUB archive: {e}")
|
||||
|
||||
# 1. Locate root OPF file from META-INF/container.xml
|
||||
opf_path = None
|
||||
try:
|
||||
container_data = z.read("META-INF/container.xml")
|
||||
root = ET.fromstring(container_data)
|
||||
rootfile = root.find(".//{*}rootfile")
|
||||
if rootfile is not None:
|
||||
opf_path = rootfile.attrib.get("full-path")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not opf_path:
|
||||
for name in z.namelist():
|
||||
if name.lower().endswith(".opf"):
|
||||
opf_path = name
|
||||
break
|
||||
|
||||
if not opf_path:
|
||||
raise ValueError("Invalid EPUB: no OPF file found")
|
||||
|
||||
opf_path = opf_path.replace("\\", "/").lstrip("/")
|
||||
opf_dir = posixpath.dirname(opf_path)
|
||||
|
||||
try:
|
||||
opf_content = z.read(opf_path)
|
||||
except KeyError:
|
||||
# Try lowercase matching
|
||||
name_map = {n.lower(): n for n in z.namelist()}
|
||||
if opf_path.lower() in name_map:
|
||||
opf_content = z.read(name_map[opf_path.lower()])
|
||||
else:
|
||||
raise ValueError(f"OPF file not found in archive: {opf_path}")
|
||||
|
||||
# 2. Extract metadata (title, author)
|
||||
title = "Untitled"
|
||||
author = ""
|
||||
opf_tree = None
|
||||
try:
|
||||
opf_tree = ET.fromstring(opf_content)
|
||||
title_el = opf_tree.find(".//{*}title")
|
||||
if title_el is not None and title_el.text:
|
||||
title = title_el.text.strip()
|
||||
creator_el = opf_tree.find(".//{*}creator")
|
||||
if creator_el is not None and creator_el.text:
|
||||
author = creator_el.text.strip()
|
||||
except Exception:
|
||||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||||
tm = re.search(r"<[^:]*:?title[^>]*>(.*?)</[^:]*:?title>", opf_str, re.I | re.S)
|
||||
if tm:
|
||||
title = re.sub(r"<[^>]+>", "", tm.group(1)).strip()
|
||||
cm = re.search(r"<[^:]*:?creator[^>]*>(.*?)</[^:]*:?creator>", opf_str, re.I | re.S)
|
||||
if cm:
|
||||
author = re.sub(r"<[^>]+>", "", cm.group(1)).strip()
|
||||
|
||||
# 3. Parse Manifest (id -> href, media-type)
|
||||
manifest: Dict[str, Dict[str, str]] = {}
|
||||
if opf_tree is not None:
|
||||
for item in opf_tree.findall(".//{*}item"):
|
||||
iid = item.attrib.get("id")
|
||||
href = item.attrib.get("href")
|
||||
media_type = item.attrib.get("media-type", "").lower()
|
||||
if iid and href:
|
||||
manifest[iid] = {"href": href, "media_type": media_type}
|
||||
else:
|
||||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||||
for m in re.finditer(r'<item\s+[^>]*id=["\']([^"\']+)["\'][^>]*href=["\']([^"\']+)["\']', opf_str, re.I):
|
||||
manifest[m.group(1)] = {"href": m.group(2), "media_type": ""}
|
||||
|
||||
# 4. Spine items in reading order
|
||||
spine_ids: List[str] = []
|
||||
if opf_tree is not None:
|
||||
for itemref in opf_tree.findall(".//{*}itemref"):
|
||||
idref = itemref.attrib.get("idref")
|
||||
if idref:
|
||||
spine_ids.append(idref)
|
||||
else:
|
||||
opf_str = opf_content.decode("utf-8", errors="ignore")
|
||||
for m in re.finditer(r'<itemref\s+[^>]*idref=["\']([^"\']+)["\']', opf_str, re.I):
|
||||
spine_ids.append(m.group(1))
|
||||
|
||||
# Fallback if spine is empty: find all html/xhtml files in manifest
|
||||
candidate_paths: List[str] = []
|
||||
if spine_ids:
|
||||
for idref in spine_ids:
|
||||
item = manifest.get(idref)
|
||||
if item:
|
||||
candidate_paths.append(item["href"])
|
||||
else:
|
||||
for item in manifest.values():
|
||||
href = item["href"]
|
||||
if href.lower().endswith((".xhtml", ".html", ".htm")):
|
||||
candidate_paths.append(href)
|
||||
|
||||
namelist_map = {name.lower(): name for name in z.namelist()}
|
||||
chapters: List[Dict[str, str]] = []
|
||||
chapter_num = 1
|
||||
|
||||
for href in candidate_paths:
|
||||
unquoted = urllib.parse.unquote(href)
|
||||
rel_path = posixpath.normpath(posixpath.join(opf_dir, unquoted)) if opf_dir else unquoted
|
||||
rel_path = rel_path.replace("\\", "/").lstrip("/")
|
||||
|
||||
actual_path = None
|
||||
if rel_path in z.namelist():
|
||||
actual_path = rel_path
|
||||
elif rel_path.lower() in namelist_map:
|
||||
actual_path = namelist_map[rel_path.lower()]
|
||||
|
||||
if not actual_path:
|
||||
continue
|
||||
|
||||
try:
|
||||
raw_bytes = z.read(actual_path)
|
||||
try:
|
||||
html_str = raw_bytes.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
html_str = raw_bytes.decode("latin-1", errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
parser = HTMLTextExtractor()
|
||||
parser.feed(html_str)
|
||||
text = parser.get_text()
|
||||
|
||||
# Skip chapters with little or no readable text
|
||||
if count_words(text) < 1:
|
||||
continue
|
||||
|
||||
ch_title = parser.heading or parser.title or f"Chapter {chapter_num}"
|
||||
# If ch_title is identical to book title or very long, format nicely
|
||||
if len(ch_title) > 120:
|
||||
ch_title = f"Chapter {chapter_num}"
|
||||
|
||||
chapters.append({"title": ch_title, "content": text})
|
||||
chapter_num += 1
|
||||
|
||||
if not chapters:
|
||||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
|
||||
def split_long_text(text: str, target_words: int = 3000) -> List[str]:
|
||||
"""
|
||||
Splits text into chunks of approximately target_words on paragraph boundaries.
|
||||
"""
|
||||
paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
|
||||
if not paragraphs:
|
||||
return [text.strip()] if text.strip() else []
|
||||
|
||||
chunks: List[str] = []
|
||||
current_paras: List[str] = []
|
||||
current_count = 0
|
||||
|
||||
for p in paragraphs:
|
||||
w_count = count_words(p)
|
||||
# If single paragraph is massive (> target_words * 1.5), split by words
|
||||
if w_count > target_words * 1.5:
|
||||
if current_paras:
|
||||
chunks.append("\n\n".join(current_paras))
|
||||
current_paras = []
|
||||
current_count = 0
|
||||
words = p.split()
|
||||
for i in range(0, len(words), target_words):
|
||||
chunk = " ".join(words[i : i + target_words])
|
||||
if chunk.strip():
|
||||
chunks.append(chunk)
|
||||
continue
|
||||
|
||||
if current_count + w_count > target_words and current_count > 0:
|
||||
chunks.append("\n\n".join(current_paras))
|
||||
current_paras = [p]
|
||||
current_count = w_count
|
||||
else:
|
||||
current_paras.append(p)
|
||||
current_count += w_count
|
||||
|
||||
if current_paras:
|
||||
chunks.append("\n\n".join(current_paras))
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||||
"""
|
||||
Parses plain text or Markdown into readable chapters.
|
||||
Splits by explicit chapter markers (e.g. 'Chapter X', '# Chapter') or chunks of ~3000 words.
|
||||
"""
|
||||
content = content.replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not content:
|
||||
return title, author, [{"title": title if title != "Untitled" else "Chapter 1", "content": ""}]
|
||||
|
||||
lines = content.split("\n")
|
||||
first_line = lines[0].strip()
|
||||
|
||||
# Extract title from first line if default
|
||||
if title == "Untitled":
|
||||
if first_line.startswith("# "):
|
||||
title = first_line.lstrip("# ").strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
elif first_line.lower().startswith("title:"):
|
||||
title = first_line.split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
|
||||
# Extract author from next line if missing
|
||||
if not author:
|
||||
lines = content.split("\n")
|
||||
if lines and lines[0].lower().startswith(("author:", "by:")):
|
||||
author = lines[0].split(":", 1)[1].strip()
|
||||
if len(lines) > 1:
|
||||
content = "\n".join(lines[1:]).strip()
|
||||
|
||||
heading_pattern = re.compile(
|
||||
r"^[ \t]*(?:"
|
||||
r"#{1,3}[ \t]+(.+)|"
|
||||
r"((?:Chapter|CHAPTER|Глава|ГЛАВА|Part|PART|Section|SECTION|Act|ACT|Scene|SCENE)[ \t]+[0-9IVXLCDMivxlcdm\w\.\-]+(?:[ \t]*[:\-–—\.][ \t]*.*|[ \t]+[^\n\r]+)?)"
|
||||
r")[ \t]*$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
matches = list(heading_pattern.finditer(content))
|
||||
chapters: List[Dict[str, str]] = []
|
||||
|
||||
if len(matches) >= 1:
|
||||
# Pre-chapter intro text
|
||||
intro = content[: matches[0].start()].strip()
|
||||
if intro and count_words(intro) > 10:
|
||||
chapters.append({"title": "Introduction", "content": intro})
|
||||
|
||||
for i, m in enumerate(matches):
|
||||
h_text = (m.group(1) or m.group(2)).strip()
|
||||
start = m.start()
|
||||
end = matches[i + 1].start() if i + 1 < len(matches) else len(content)
|
||||
body = content[start:end].strip()
|
||||
if body:
|
||||
chapters.append({"title": h_text, "content": body})
|
||||
|
||||
if not chapters:
|
||||
# No headings detected: split by ~3000 words chunks
|
||||
chunks = split_long_text(content, target_words=3000)
|
||||
if len(chunks) == 1:
|
||||
chapters = [{"title": title if title != "Untitled" else "Chapter 1", "content": chunks[0]}]
|
||||
else:
|
||||
for idx, chunk in enumerate(chunks, start=1):
|
||||
chapters.append({"title": f"Chapter {idx}", "content": chunk})
|
||||
|
||||
return title, author, chapters
|
||||
Reference in New Issue
Block a user