feat(reader): add PDF reading support with outline chapters, CLI blocks, and noise filtering (task 7)
This commit is contained in:
+317
@@ -1,9 +1,11 @@
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
import html
|
||||
import zipfile
|
||||
import posixpath
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
import xml.etree.ElementTree as ET
|
||||
from html.parser import HTMLParser
|
||||
from typing import Tuple, List, Dict
|
||||
@@ -496,3 +498,318 @@ def parse_text(content: str, title: str = "Untitled", author: str = "") -> Tuple
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF Parsing & Chapter Extraction (pypdf, Bookmarks/Outline, Cisco CLI Blocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLI_PROMPT_RE = re.compile(
|
||||
r"^(?:[A-Za-z0-9_.\-]+(?:\([a-zA-Z0-9_\-\.\s]+\))?[>#]|"
|
||||
r"[A-Za-z0-9_.\-]+@[A-Za-z0-9_.\-]+:[~/\w\.\-]*[$#]|"
|
||||
r"(?:[A-Z]):\\[^>\n\r]*>|"
|
||||
r"PS\s+[A-Z]:\\[^>\n\r]*>)"
|
||||
)
|
||||
|
||||
|
||||
def extract_outline_entries(outline, reader) -> List[Tuple[str, int]]:
|
||||
"""
|
||||
Recursively extracts bookmarks / outline entries from PDF.
|
||||
Returns a list of (title, 0-indexed page_number).
|
||||
"""
|
||||
entries: List[Tuple[str, int]] = []
|
||||
|
||||
def _traverse(items):
|
||||
for item in items:
|
||||
if isinstance(item, list):
|
||||
_traverse(item)
|
||||
else:
|
||||
try:
|
||||
title = getattr(item, "title", None) or (item.get("/Title") if hasattr(item, "get") else None)
|
||||
page_num = reader.get_destination_page_number(item)
|
||||
if title and page_num is not None:
|
||||
clean_title = str(title).strip()
|
||||
if clean_title:
|
||||
entries.append((clean_title, int(page_num)))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if outline:
|
||||
_traverse(outline)
|
||||
return entries
|
||||
|
||||
|
||||
def clean_page_lines(raw_pages_text: List[str]) -> List[str]:
|
||||
"""
|
||||
Filters running headers, footers, isolated page numbers, and copyright noise.
|
||||
"""
|
||||
if not raw_pages_text:
|
||||
return []
|
||||
|
||||
num_pages = len(raw_pages_text)
|
||||
top_candidates: Dict[str, int] = {}
|
||||
bottom_candidates: Dict[str, int] = {}
|
||||
|
||||
for pt in raw_pages_text:
|
||||
lines = [l.strip() for l in pt.split("\n") if l.strip()]
|
||||
if not lines:
|
||||
continue
|
||||
if len(lines) >= 1:
|
||||
top_candidates[lines[0]] = top_candidates.get(lines[0], 0) + 1
|
||||
bottom_candidates[lines[-1]] = bottom_candidates.get(lines[-1], 0) + 1
|
||||
|
||||
# Lines repeated on >= 20% of pages (min 3 pages) are running headers/footers
|
||||
threshold = max(3, int(num_pages * 0.20))
|
||||
common_headers = {l for l, c in top_candidates.items() if c >= threshold}
|
||||
common_footers = {l for l, c in bottom_candidates.items() if c >= threshold}
|
||||
|
||||
page_num_re = re.compile(r"^(?:page\s+)?\d+(?:\s*(?:of|/)\s*\d+)?$", re.IGNORECASE)
|
||||
url_re = re.compile(r"^(?:https?://|www\.)\S+$", re.IGNORECASE)
|
||||
copyright_re = re.compile(r"^(?:©|\(c\)|copyright)\s*\d{4}.*$", re.IGNORECASE)
|
||||
|
||||
cleaned_pages: List[str] = []
|
||||
for pt in raw_pages_text:
|
||||
lines = pt.split("\n")
|
||||
filtered: List[str] = []
|
||||
n_lines = len(lines)
|
||||
for idx, l in enumerate(lines):
|
||||
stripped = l.strip()
|
||||
if not stripped:
|
||||
filtered.append("")
|
||||
continue
|
||||
is_top = (idx <= 2)
|
||||
is_bottom = (idx >= n_lines - 3)
|
||||
|
||||
if (is_top or is_bottom) and page_num_re.match(stripped):
|
||||
continue
|
||||
if is_top and stripped in common_headers:
|
||||
continue
|
||||
if is_bottom and (stripped in common_footers or url_re.match(stripped) or copyright_re.match(stripped)):
|
||||
continue
|
||||
filtered.append(l)
|
||||
cleaned_pages.append("\n".join(filtered))
|
||||
|
||||
return cleaned_pages
|
||||
|
||||
|
||||
def format_cli_blocks(text: str) -> str:
|
||||
"""
|
||||
Detects Cisco IOS / CLI terminal sessions in text and wraps them
|
||||
in Markdown fenced code blocks (```text ... ```) for syntax highlighting and copying.
|
||||
"""
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
lines = text.split("\n")
|
||||
out: List[str] = []
|
||||
in_cli = False
|
||||
cli_lines: List[str] = []
|
||||
|
||||
def is_likely_cli_line(line: str) -> bool:
|
||||
s = line.strip()
|
||||
if not s:
|
||||
return True
|
||||
if CLI_PROMPT_RE.match(s):
|
||||
return True
|
||||
if s.startswith(("%", "!", "--", ">>", "Enter configuration", "Building configuration", "Current configuration")):
|
||||
return True
|
||||
if len(re.findall(r"\S\s{2,}\S", line)) >= 2:
|
||||
return True
|
||||
if re.match(r"^(?:show|ip|no|interface|switchport|vlan|router|network|exit|end|enable|disable|configure|ping|traceroute|debug)\s+", s, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
for line in lines:
|
||||
s = line.strip()
|
||||
if not in_cli:
|
||||
if CLI_PROMPT_RE.match(s):
|
||||
in_cli = True
|
||||
cli_lines = [line]
|
||||
else:
|
||||
out.append(line)
|
||||
else:
|
||||
if is_likely_cli_line(line):
|
||||
cli_lines.append(line)
|
||||
else:
|
||||
while cli_lines and not cli_lines[-1].strip():
|
||||
cli_lines.pop()
|
||||
if cli_lines:
|
||||
out.append("```text")
|
||||
out.extend(cli_lines)
|
||||
out.append("```")
|
||||
in_cli = False
|
||||
cli_lines = []
|
||||
out.append(line)
|
||||
|
||||
if in_cli and cli_lines:
|
||||
while cli_lines and not cli_lines[-1].strip():
|
||||
cli_lines.pop()
|
||||
if cli_lines:
|
||||
out.append("```text")
|
||||
out.extend(cli_lines)
|
||||
out.append("```")
|
||||
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def parse_pdf(file_bytes: bytes, filename: str = "") -> Tuple[str, str, List[Dict[str, str]]]:
|
||||
"""
|
||||
Parses a PDF file into readable chapters:
|
||||
- Extracts title and author from metadata or filename.
|
||||
- Enforces Scanned PDF Guard (raises ValueError if total_words < 100).
|
||||
- Cleans repetitive headers, footers, and page numbers.
|
||||
- Wraps Cisco IOS / CLI terminal blocks in Markdown code blocks.
|
||||
- Splits chapters by PDF outline / bookmarks (Cisco Press, O'Reilly).
|
||||
- Fallback to heading regex or ~10-15 page chunks if no bookmarks exist.
|
||||
"""
|
||||
try:
|
||||
import pypdf
|
||||
except ImportError:
|
||||
vendor_dir = Path(__file__).resolve().parent / "vendor"
|
||||
if str(vendor_dir) not in sys.path:
|
||||
sys.path.insert(0, str(vendor_dir))
|
||||
import pypdf
|
||||
|
||||
try:
|
||||
reader = pypdf.PdfReader(io.BytesIO(file_bytes))
|
||||
except Exception as e:
|
||||
raise ValueError(f"Не удалось открыть PDF документ: {e}")
|
||||
|
||||
if reader.is_encrypted:
|
||||
try:
|
||||
reader.decrypt("")
|
||||
except Exception:
|
||||
raise ValueError("Этот PDF документ защищен паролем. Пожалуйста, снимите пароль для чтения.")
|
||||
|
||||
num_pages = len(reader.pages)
|
||||
if num_pages == 0:
|
||||
raise ValueError("PDF документ не содержит страниц")
|
||||
|
||||
# Metadata extraction
|
||||
meta = reader.metadata or {}
|
||||
meta_title = getattr(meta, "title", None) or (meta.get("/Title") if hasattr(meta, "get") else None) or ""
|
||||
meta_author = getattr(meta, "author", None) or (meta.get("/Author") if hasattr(meta, "get") else None) or ""
|
||||
|
||||
stem = Path(filename).stem if filename else "Untitled Document"
|
||||
clean_title = str(meta_title).strip()
|
||||
if not clean_title or clean_title.lower() in {"untitled", "untitled document", "default", "temp", "output"}:
|
||||
title = stem
|
||||
else:
|
||||
title = clean_title
|
||||
|
||||
author = str(meta_author).strip()
|
||||
|
||||
# Page-by-page text extraction
|
||||
raw_pages_text: List[str] = []
|
||||
for p in reader.pages:
|
||||
try:
|
||||
pt = p.extract_text() or ""
|
||||
except Exception:
|
||||
pt = ""
|
||||
raw_pages_text.append(pt)
|
||||
|
||||
# Scanned PDF Guard (< 100 words)
|
||||
total_words = sum(count_words(t) for t in raw_pages_text)
|
||||
if total_words < 100:
|
||||
raise ValueError(
|
||||
"Этот PDF-документ содержит сканы или изображения без текстового слоя (OCR). "
|
||||
"Пожалуйста, используйте документ с распознанным текстом."
|
||||
)
|
||||
|
||||
# Clean noise lines (headers, footers, page numbers)
|
||||
cleaned_pages = clean_page_lines(raw_pages_text)
|
||||
|
||||
# Apply CLI formatting to each page
|
||||
formatted_pages = [format_cli_blocks(p) for p in cleaned_pages]
|
||||
|
||||
# Chapter splitting:
|
||||
# 1. Try outline / bookmarks
|
||||
outline_entries = extract_outline_entries(getattr(reader, "outline", None), reader)
|
||||
chapters: List[Dict[str, str]] = []
|
||||
|
||||
if outline_entries:
|
||||
valid_entries = [(t, p) for t, p in outline_entries if 0 <= p < num_pages]
|
||||
valid_entries.sort(key=lambda x: x[1])
|
||||
|
||||
# Filter duplicates pointing to the same page
|
||||
deduped_entries: List[Tuple[str, int]] = []
|
||||
for t, p in valid_entries:
|
||||
if not deduped_entries or deduped_entries[-1][1] != p:
|
||||
deduped_entries.append((t, p))
|
||||
|
||||
if deduped_entries:
|
||||
first_p = deduped_entries[0][1]
|
||||
if first_p > 0:
|
||||
intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, first_p) if formatted_pages[p].strip())
|
||||
if count_words(intro_content) >= 20:
|
||||
is_ru = any(ord(c) > 127 for c in intro_content[:300])
|
||||
chapters.append({
|
||||
"title": "Введение" if is_ru else "Introduction",
|
||||
"content": intro_content
|
||||
})
|
||||
|
||||
for i, (entry_title, start_p) in enumerate(deduped_entries):
|
||||
end_p = deduped_entries[i + 1][1] if i + 1 < len(deduped_entries) else num_pages
|
||||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||||
if ch_content:
|
||||
chapters.append({
|
||||
"title": entry_title,
|
||||
"content": ch_content
|
||||
})
|
||||
|
||||
# 2. Fallback: No outline found, check for page-level chapter headings
|
||||
if not chapters:
|
||||
chapter_heading_re = re.compile(
|
||||
r"^\s*(?:chapter|глава|part|часть|section|раздел|module|модуль)\s+([0-9ivxlcdm]+)[\s:.\-–—\n]+(.*)$",
|
||||
re.IGNORECASE
|
||||
)
|
||||
detected_pages: List[Tuple[str, int]] = []
|
||||
for p_idx, p_text in enumerate(formatted_pages):
|
||||
first_lines = [l.strip() for l in p_text.split("\n") if l.strip()][:3]
|
||||
for l in first_lines:
|
||||
m = chapter_heading_re.match(l)
|
||||
if m:
|
||||
h_title = l.strip("#").strip()
|
||||
detected_pages.append((h_title, p_idx))
|
||||
break
|
||||
|
||||
if detected_pages:
|
||||
if detected_pages[0][1] > 0:
|
||||
intro_content = "\n\n".join(formatted_pages[p].strip() for p in range(0, detected_pages[0][1]) if formatted_pages[p].strip())
|
||||
if count_words(intro_content) >= 20:
|
||||
is_ru = any(ord(c) > 127 for c in intro_content[:300])
|
||||
chapters.append({
|
||||
"title": "Введение" if is_ru else "Introduction",
|
||||
"content": intro_content
|
||||
})
|
||||
|
||||
for i, (h_title, start_p) in enumerate(detected_pages):
|
||||
end_p = detected_pages[i + 1][1] if i + 1 < len(detected_pages) else num_pages
|
||||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||||
if ch_content:
|
||||
chapters.append({
|
||||
"title": h_title,
|
||||
"content": ch_content
|
||||
})
|
||||
|
||||
# 3. Fallback: Group pages in chunks of 12-15 pages
|
||||
if not chapters:
|
||||
chunk_size = 12
|
||||
sample_text = "".join(formatted_pages[:3])
|
||||
is_ru = any(ord(c) > 127 for c in sample_text[:300])
|
||||
prefix = "Страницы" if is_ru else "Pages"
|
||||
|
||||
for start_p in range(0, num_pages, chunk_size):
|
||||
end_p = min(start_p + chunk_size, num_pages)
|
||||
ch_content = "\n\n".join(formatted_pages[p].strip() for p in range(start_p, end_p) if formatted_pages[p].strip())
|
||||
if ch_content:
|
||||
chapters.append({
|
||||
"title": f"{prefix} {start_p + 1}–{end_p}",
|
||||
"content": ch_content
|
||||
})
|
||||
|
||||
if not chapters:
|
||||
full_content = "\n\n".join(p.strip() for p in formatted_pages if p.strip())
|
||||
chapters = [{"title": title, "content": full_content}]
|
||||
|
||||
return title, author, chapters
|
||||
|
||||
|
||||
Reference in New Issue
Block a user