""" Phrasal Verb & Multi-Word Collocation Detector. Analyzes context sentence tokens around a clicked word to identify phrasal verbs (e.g. 'take off', 'look forward to', 'turn it off') and idioms. 100% offline, zero-dependency, pure Python stdlib + SQLite. """ import re import sqlite3 from typing import Optional, Dict, Any, List from app.database import get_db IRREGULARS = { "took": "take", "taken": "take", "takes": "take", "taking": "take", "gave": "give", "given": "give", "gives": "give", "giving": "give", "got": "get", "gotten": "get", "gets": "get", "getting": "get", "went": "go", "gone": "go", "goes": "go", "going": "go", "came": "come", "comes": "come", "coming": "come", "made": "make", "makes": "make", "making": "make", "put": "put", "puts": "put", "putting": "put", "ran": "run", "runs": "run", "running": "run", "held": "hold", "holds": "hold", "holding": "hold", "brought": "bring", "brings": "bring", "bringing": "bring", "broke": "break", "broken": "break", "breaks": "break", "breaking": "break", "fell": "fall", "fallen": "fall", "falls": "fall", "falling": "fall", "kept": "keep", "keeps": "keep", "keeping": "keep", "stood": "stand", "stands": "stand", "standing": "stand", "drew": "draw", "drawn": "draw", "draws": "draw", "drawing": "draw", "found": "find", "finds": "find", "finding": "find", "blew": "blow", "blown": "blow", "blows": "blow", "blowing": "blow", "hung": "hang", "hangs": "hang", "hanging": "hang", "caught": "catch", "catches": "catch", "catching": "catch", "threw": "throw", "thrown": "throw", "throws": "throw", "throwing": "throw", "told": "tell", "tells": "tell", "telling": "tell", "left": "leave", "leaves": "leave", "leaving": "leave", "grew": "grow", "grown": "grow", "grows": "grow", "growing": "grow", "paid": "pay", "pays": "pay", "paying": "pay", "wound": "wind", "winds": "wind", "winding": "wind", "built": "build", "builds": "build", "building": "build", "cut": "cut", "cuts": "cut", "cutting": "cut", "shut": "shut", "shuts": "shut", "shutting": "shut", "set": "set", "sets": "set", "setting": "set", "split": "split", "splits": "split", "splitting": "split", "spread": "spread", "spreads": "spread", "spreading": "spread", } PRONOUNS_AND_OBJECTS = { "it", "him", "her", "them", "us", "me", "you", "this", "that", "these", "those", "things", "something", "everything", "anything", "all" } PHRASAL_PARTICLES = { "about", "across", "ahead", "along", "apart", "around", "round", "aside", "at", "away", "back", "by", "down", "for", "forth", "forward", "in", "into", "off", "on", "onto", "out", "over", "past", "round", "through", "to", "together", "under", "up", "upon", "with" } import json from pathlib import Path from app import config _CURATED_PHRASAL_VERBS_CACHE: Optional[Dict[str, str]] = None def get_curated_phrasal_verbs() -> Dict[str, str]: global _CURATED_PHRASAL_VERBS_CACHE if _CURATED_PHRASAL_VERBS_CACHE is None: seeds_file = config.BASE_DIR / "data" / "seeds" / "phrasal_verbs.json" if seeds_file.is_file(): try: with open(seeds_file, "r", encoding="utf-8") as f: _CURATED_PHRASAL_VERBS_CACHE = json.load(f) except Exception: _CURATED_PHRASAL_VERBS_CACHE = {} else: _CURATED_PHRASAL_VERBS_CACHE = {} return _CURATED_PHRASAL_VERBS_CACHE CURATED_PHRASAL_VERBS: Dict[str, str] = get_curated_phrasal_verbs() def lemmatize_word(w: str) -> List[str]: """Generates candidate base forms for English verbs and nouns.""" w = w.lower() res = [w] if w in IRREGULARS: res.append(IRREGULARS[w]) if w.endswith("ing") and len(w) > 4: stem = w[:-3] res.extend([stem, stem + "e"]) if len(stem) > 2 and stem[-1] == stem[-2]: res.append(stem[:-1]) elif w.endswith("ed") and len(w) > 3: stem = w[:-2] res.extend([stem, stem + "e"]) if stem.endswith("i"): res.append(stem[:-1] + "y") if len(stem) > 2 and stem[-1] == stem[-2]: res.append(stem[:-1]) elif w.endswith("s") and len(w) > 3: if w.endswith("es"): res.extend([w[:-2], w[:-1]]) if w.endswith("ies"): res.append(w[:-3] + "y") else: res.append(w[:-1]) # De-duplicate preserving order return list(dict.fromkeys(res)) def detect_phrasal_verb( sentence: str, target_word: str, user_id: Optional[int] = None, conn: Optional[sqlite3.Connection] = None, ) -> Optional[Dict[str, Any]]: """ Detects if target_word in sentence is part of a phrasal verb or collocation. Checks window sizes 4, 3, 2 and separable phrasal verbs. Returns matched term, raw snippet in text, translation, separable flag, and library status. """ if not sentence or not target_word: return None tokens = re.findall(r"\b[a-zA-Z\'-]+\b", sentence) lower_tokens = [t.lower() for t in tokens] target_lower = target_word.strip().lower() target_indices = [i for i, t in enumerate(lower_tokens) if t == target_lower] if not target_indices: return None def execute_query(query: str, params: tuple): if conn is not None: return conn.execute(query, params).fetchone() with get_db() as c: return c.execute(query, params).fetchone() match_result = None for idx in target_indices: # 1. Contiguous spans: 4, 3, 2 words containing target_word for span_len in (4, 3, 2): for start in range(max(0, idx - span_len + 1), min(len(tokens) - span_len + 1, idx + 1)): span_tokens = lower_tokens[start : start + span_len] raw_match = " ".join(tokens[start : start + span_len]) v_lemmas = lemmatize_word(span_tokens[0]) for v in v_lemmas: cand = " ".join([v] + span_tokens[1:]) if cand in CURATED_PHRASAL_VERBS: match_result = { "term": cand, "raw": raw_match, "translation": CURATED_PHRASAL_VERBS[cand], "separable": False, "matched_words": tokens[start : start + span_len], } break # Check offline dictionary only for genuine phrasal verbs: # 1. Candidate must end with a known particle (e.g. 'turn on', 'come up with') # 2. pos in offline_dictionary must explicitly be a verb / verb phrase cand_words = cand.split() if len(cand_words) >= 2 and cand_words[-1] in PHRASAL_PARTICLES: row = execute_query( "SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (cand,), ) if row: pos_val = (row[2] or "").lower() if "verb" in pos_val: match_result = { "term": row[0], "raw": raw_match, "translation": row[1], "separable": False, "matched_words": tokens[start : start + span_len], } break if match_result: break if match_result: break if match_result: break # 2. Separable phrasal verbs: verb + pronoun/short object + particle (e.g. turned it off -> turn off) for v_idx in range(max(0, idx - 2), min(len(tokens) - 2, idx + 1)): if lower_tokens[v_idx + 1] in PRONOUNS_AND_OBJECTS: p_idx = v_idx + 2 if idx in (v_idx, p_idx): particle = lower_tokens[p_idx] if particle not in PHRASAL_PARTICLES: continue v_lemmas = lemmatize_word(lower_tokens[v_idx]) raw_match = " ".join(tokens[v_idx : p_idx + 1]) for v in v_lemmas: cand = f"{v} {particle}" if cand in CURATED_PHRASAL_VERBS: match_result = { "term": cand, "raw": raw_match, "translation": CURATED_PHRASAL_VERBS[cand], "separable": True, "matched_words": [tokens[v_idx], tokens[p_idx]], } break row = execute_query( "SELECT term, translation, pos FROM offline_dictionary WHERE term = ? COLLATE NOCASE", (cand,), ) if row: pos_val = (row[2] or "").lower() if "verb" in pos_val: match_result = { "term": row[0], "raw": raw_match, "translation": row[1], "separable": True, "matched_words": [tokens[v_idx], tokens[p_idx]], } break if match_result: break if match_result: break if not match_result: return None # Check if this phrasal verb already exists in user's vocabulary match_result["in_library"] = False match_result["library_card"] = None if user_id is not None: row = execute_query( "SELECT id, term, translation, context_sentence, tags, collection, srs_state, srs_reps " "FROM vocabulary_cards WHERE user_id = ? AND lower(trim(term)) = lower(trim(?))", (user_id, match_result["term"]), ) if row: match_result["in_library"] = True match_result["library_card"] = dict(row) return match_result def seed_curated_phrasal_verbs(conn: sqlite3.Connection): """Ensures all curated phrasal verbs are present in offline_dictionary table.""" try: for term, trans in CURATED_PHRASAL_VERBS.items(): conn.execute( """ INSERT OR IGNORE INTO offline_dictionary (term, translation, phonetic, pos) VALUES (?, ?, '', 'verb phrase') """, (term, trans), ) except Exception: pass