107 lines
3.8 KiB
Python
107 lines
3.8 KiB
Python
|
"""Text normalization for Swedish recipe content.
|
|||
|
|
|
|||
|
|
Pure functions, no I/O. These encode pitfalls verified against Fredrik's live
|
|||
|
|
Mealie instance — see resources/import-pitfalls.md.
|
|||
|
|
"""
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
#: Campaign/navigation suffixes seen appended to imported titles, notably from Köket.
|
|||
|
|
TITLE_JUNK_SUFFIXES = (
|
|||
|
|
"- se & gör",
|
|||
|
|
"- se och gör",
|
|||
|
|
"| Köket.se",
|
|||
|
|
"| Köket",
|
|||
|
|
"- Recept",
|
|||
|
|
"- recept",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
#: Placeholder text Mealie writes when a scrape fails to find real content.
|
|||
|
|
EXTRACTION_FAILURE_MARKERS = (
|
|||
|
|
"Could not detect ingredients",
|
|||
|
|
"Could not detect instructions",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
#: English units/labels that must not survive into a finished Swedish recipe.
|
|||
|
|
ENGLISH_LEFTOVERS = (
|
|||
|
|
"cup", "cups", "tbsp", "tsp", "tablespoon", "teaspoon",
|
|||
|
|
"oz", "ounce", "lb", "pound", "clove", "cloves",
|
|||
|
|
"servings", "ingredients", "instructions",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
_DECIMAL_COMMA = re.compile(r"(?<=\d),(?=\d)")
|
|||
|
|
_WHITESPACE = re.compile(r"\s+")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_for_parser(line: str) -> str:
|
|||
|
|
"""Return a variant of a Swedish ingredient line the Mealie parser handles better.
|
|||
|
|
|
|||
|
|
Only converts decimal comma to decimal point (``4,7 dl`` -> ``4.7 dl``).
|
|||
|
|
The result is for the parser only; the original line stays canonical for
|
|||
|
|
``display``/``note``, because the parser mangles Swedish text.
|
|||
|
|
"""
|
|||
|
|
return _WHITESPACE.sub(" ", _DECIMAL_COMMA.sub(".", line)).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def clean_title(title: str) -> str:
|
|||
|
|
"""Strip campaign/navigation junk that import scrapers append to recipe titles."""
|
|||
|
|
cleaned = title.strip()
|
|||
|
|
changed = True
|
|||
|
|
while changed:
|
|||
|
|
changed = False
|
|||
|
|
for suffix in TITLE_JUNK_SUFFIXES:
|
|||
|
|
if cleaned.lower().endswith(suffix.lower()):
|
|||
|
|
cleaned = cleaned[: -len(suffix)].strip(" -|–")
|
|||
|
|
changed = True
|
|||
|
|
return _WHITESPACE.sub(" ", cleaned).strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def slugify(title: str) -> str:
|
|||
|
|
"""Build a Mealie-style slug from a Swedish title, preserving å/ä/ö transliteration."""
|
|||
|
|
text = title.lower()
|
|||
|
|
for source, target in (("å", "a"), ("ä", "a"), ("ö", "o"), ("é", "e"), ("&", "och")):
|
|||
|
|
text = text.replace(source, target)
|
|||
|
|
text = re.sub(r"[^a-z0-9]+", "-", text)
|
|||
|
|
return text.strip("-")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def find_english_leftovers(text: str) -> list[str]:
|
|||
|
|
"""Return English unit/label words still present in supposedly Swedish content."""
|
|||
|
|
lowered = text.lower()
|
|||
|
|
return [word for word in ENGLISH_LEFTOVERS if re.search(rf"\b{re.escape(word)}\b", lowered)]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def has_extraction_failure(recipe: dict) -> bool:
|
|||
|
|
"""Detect Mealie's 'could not detect' placeholders anywhere in a recipe."""
|
|||
|
|
haystack = [recipe.get("description") or ""]
|
|||
|
|
haystack += [i.get("display") or i.get("note") or "" for i in recipe.get("recipeIngredient") or []]
|
|||
|
|
haystack += [s.get("text") or "" for s in recipe.get("recipeInstructions") or []]
|
|||
|
|
blob = " ".join(haystack)
|
|||
|
|
return any(marker in blob for marker in EXTRACTION_FAILURE_MARKERS)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def ingredient_display_lines(recipe: dict) -> list[str]:
|
|||
|
|
"""Extract the human-facing ingredient lines, preferring display over note."""
|
|||
|
|
lines = []
|
|||
|
|
for item in recipe.get("recipeIngredient") or []:
|
|||
|
|
text = item.get("display") or item.get("note") or ""
|
|||
|
|
if not text:
|
|||
|
|
food = item.get("food") or {}
|
|||
|
|
text = food.get("name") or ""
|
|||
|
|
lines.append(text)
|
|||
|
|
return lines
|
|||
|
|
|
|||
|
|
|
|||
|
|
def looks_mangled(line: str) -> bool:
|
|||
|
|
"""Heuristic for parser-damaged display text.
|
|||
|
|
|
|||
|
|
Verified failure modes: vulgar/composed fractions (``47⁄10 liter``), duplicated
|
|||
|
|
units, and stray fraction glyphs the Mealie parser emits for Swedish rows.
|
|||
|
|
"""
|
|||
|
|
if any(glyph in line for glyph in ("⁄", "½", "¼", "¾", "⅓", "⅔")):
|
|||
|
|
return True
|
|||
|
|
if re.search(r"\b(\w+)\s+\1\b", line, flags=re.IGNORECASE):
|
|||
|
|
return True
|
|||
|
|
return False
|