ecki/eventlens
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mojibake bereinigt

Browse Source
This commit is contained in:
ecki
2026-05-02 17:39:24 +02:00
parent 18462f97e8
commit 27158071f9
1 changed files with 41 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/source_scanner.py
+41 -24
View File
@@ -55,6 +55,23 @@ FOLLOW_LINK_KEYWORDS = (
MAX_FOLLOWED_LINKS = 6 MAX_FOLLOWED_LINKS = 6
MOJIBAKE_MARKERS = ("Â", "Ã", "â", *[chr(code) for code in range(0x80, 0xA0)])
def clean_display_text(value: str | None) -> str:
if not value:
return ""
cleaned = " ".join(str(value).split())
if not any(marker in cleaned for marker in MOJIBAKE_MARKERS):
return cleaned
try:
repaired = cleaned.encode("latin1").decode("utf-8")
except UnicodeError:
return cleaned
return " ".join(repaired.split())
class SourceScanner: class SourceScanner:
headers = { headers = {
@@ -199,7 +216,7 @@ class SourceScanner:
past_count = 0 past_count = 0
for event in events: for event in events:
title = event.get("name") or "" title = clean_display_text(event.get("name"))
performers = self._extract_performer_names(event) performers = self._extract_performer_names(event)
haystack = normalize_search_text(" ".join([title] + performers)) haystack = normalize_search_text(" ".join([title] + performers))
if not self._term_matches_normalized(normalized_term, haystack): if not self._term_matches_normalized(normalized_term, haystack):
@@ -208,7 +225,7 @@ class SourceScanner:
location = event.get("location") or {} location = event.get("location") or {}
address = location.get("address") or {} address = location.get("address") or {}
city = address.get("addressLocality") or location.get("name") city = clean_display_text(address.get("addressLocality") or location.get("name"))
if watch_item.region_scope == RegionScope.hamburg and normalize_search_text(city) != "hamburg": if watch_item.region_scope == RegionScope.hamburg and normalize_search_text(city) != "hamburg":
outside_region_count += 1 outside_region_count += 1
continue continue
@@ -224,7 +241,7 @@ class SourceScanner:
"external_id": str(event.get("@id") or ticket_url or f"{source.id}:{title}"), "external_id": str(event.get("@id") or ticket_url or f"{source.id}:{title}"),
"title": title or watch_item.name, "title": title or watch_item.name,
"matched_term": watch_item.name, "matched_term": watch_item.name,
"venue_name": location.get("name") or source.label, "venue_name": clean_display_text(location.get("name") or source.label),
"city": city, "city": city,
"country_code": "DE", "country_code": "DE",
"event_date": event_date, "event_date": event_date,
@@ -279,7 +296,7 @@ class SourceScanner:
past_contexts = 0 past_contexts = 0
outside_region_contexts = 0 outside_region_contexts = 0
for context in self._find_matching_contexts(soup, watch_item): for context in self._find_matching_contexts(soup, watch_item):
context_text = context.get_text(" ", strip=True) context_text = clean_display_text(context.get_text(" ", strip=True))
event_date = self._find_nearest_date(context_text, watch_item.name) event_date = self._find_nearest_date(context_text, watch_item.name)
if event_date is None: if event_date is None:
continue continue
@@ -307,7 +324,7 @@ class SourceScanner:
"external_id": key, "external_id": key,
"title": title, "title": title,
"matched_term": watch_item.name, "matched_term": watch_item.name,
"venue_name": self._find_venue(context_text, source.label), "venue_name": clean_display_text(self._find_venue(context_text, source.label)),
"city": "Hamburg" if watch_item.region_scope == RegionScope.hamburg else None, "city": "Hamburg" if watch_item.region_scope == RegionScope.hamburg else None,
"country_code": "DE", "country_code": "DE",
"event_date": event_date, "event_date": event_date,
@@ -316,7 +333,7 @@ class SourceScanner:
"raw_payload": { "raw_payload": {
"source_url": context_url, "source_url": context_url,
"parser": "html_text", "parser": "html_text",
"context": context_text[:1000], "context": clean_display_text(context_text[:1000]),
}, },
} }
) )
@@ -372,8 +389,8 @@ class SourceScanner:
): ):
continue continue
city = self._find_city_in_line(line, watch_item.region_scope) city = clean_display_text(self._find_city_in_line(line, watch_item.region_scope))
venue_name = self._find_venue_from_line(line, source.label) venue_name = clean_display_text(self._find_venue_from_line(line, source.label))
key = f"{source.id}:{normalize_search_text(line)}:{event_date.date().isoformat()}" key = f"{source.id}:{normalize_search_text(line)}:{event_date.date().isoformat()}"
if key in seen_keys: if key in seen_keys:
continue continue
@@ -393,7 +410,7 @@ class SourceScanner:
"raw_payload": { "raw_payload": {
"source_url": context_url, "source_url": context_url,
"parser": "html_line", "parser": "html_line",
"context": line[:1000], "context": clean_display_text(line[:1000]),
}, },
} }
) )
@@ -404,7 +421,7 @@ class SourceScanner:
lines = [] lines = []
seen_lines: set[str] = set() seen_lines: set[str] = set()
for line in soup.get_text("\n", strip=True).splitlines(): for line in soup.get_text("\n", strip=True).splitlines():
cleaned = " ".join(line.split()) cleaned = clean_display_text(line)
if len(cleaned) < 8: if len(cleaned) < 8:
continue continue
if not self._find_nearest_date(cleaned, ""): if not self._find_nearest_date(cleaned, ""):
@@ -417,7 +434,7 @@ class SourceScanner:
for node in soup.find_all( for node in soup.find_all(
["article", "li", "p", "tr", "section", "div", "span", "h1", "h2", "h3", "h4"] ["article", "li", "p", "tr", "section", "div", "span", "h1", "h2", "h3", "h4"]
): ):
cleaned = " ".join(node.get_text(" ", strip=True).split()) cleaned = clean_display_text(node.get_text(" ", strip=True))
if len(cleaned) < 8 or len(cleaned) > 500: if len(cleaned) < 8 or len(cleaned) > 500:
continue continue
if not self._find_nearest_date(cleaned, ""): if not self._find_nearest_date(cleaned, ""):
@@ -438,14 +455,14 @@ class SourceScanner:
re.IGNORECASE, re.IGNORECASE,
) )
if match: if match:
return match.group(1).strip(" /-") return clean_display_text(match.group(1).strip(" /-"))
return None return None
def _find_venue_from_line(self, line: str, default: str) -> str: def _find_venue_from_line(self, line: str, default: str) -> str:
if "/" not in line: if "/" not in line:
return default return clean_display_text(default)
venue = line.split("/", 1)[1].strip() venue = clean_display_text(line.split("/", 1)[1].strip())
return venue or default return venue or clean_display_text(default)
def _find_link_for_line(self, soup: BeautifulSoup, line: str, base_url: str) -> str | None: def _find_link_for_line(self, soup: BeautifulSoup, line: str, base_url: str) -> str | None:
normalized_line = normalize_search_text(line) normalized_line = normalize_search_text(line)
@@ -559,9 +576,9 @@ class SourceScanner:
def _extract_performer_names(self, event: dict) -> list[str]: def _extract_performer_names(self, event: dict) -> list[str]:
performer = event.get("performer") or event.get("performers") performer = event.get("performer") or event.get("performers")
if isinstance(performer, dict): if isinstance(performer, dict):
return [performer.get("name", "")] return [clean_display_text(performer.get("name"))]
if isinstance(performer, list): if isinstance(performer, list):
return [item.get("name", "") for item in performer if isinstance(item, dict)] return [clean_display_text(item.get("name")) for item in performer if isinstance(item, dict)]
return [] return []
def _extract_image(self, event: dict) -> str | None: def _extract_image(self, event: dict) -> str | None:
@@ -744,8 +761,8 @@ class SourceScanner:
for line in lines: for line in lines:
normalized = normalize_search_text(line) normalized = normalize_search_text(line)
if "hamburg" in normalized and len(line) <= 120: if "hamburg" in normalized and len(line) <= 120:
return line return clean_display_text(line)
return default return clean_display_text(default)
def _find_best_context(self, soup: BeautifulSoup, term: str): def _find_best_context(self, soup: BeautifulSoup, term: str):
normalized_term = normalize_search_text(term) normalized_term = normalize_search_text(term)
@@ -789,11 +806,11 @@ class SourceScanner:
return term return term
normalized_term = normalize_search_text(term) normalized_term = normalize_search_text(term)
for heading in soup.find_all(["h1", "h2", "h3", "h4", "strong", "b", "a"]): for heading in soup.find_all(["h1", "h2", "h3", "h4", "strong", "b", "a"]):
title = heading.get_text(" ", strip=True) title = clean_display_text(heading.get_text(" ", strip=True))
if self._term_matches_normalized(normalized_term, normalize_search_text(title)): if self._term_matches_normalized(normalized_term, normalize_search_text(title)):
return title return title
text = soup.get_text(" ", strip=True) text = clean_display_text(soup.get_text(" ", strip=True))
dated_match = re.search( dated_match = re.search(
r"(.{0,40}\d{1,2}\.\d{1,2}\.(?:\d{2,4})?.{0,100}" r"(.{0,40}\d{1,2}\.\d{1,2}\.(?:\d{2,4})?.{0,100}"
+ re.escape(term) + re.escape(term)
@@ -802,12 +819,12 @@ class SourceScanner:
re.IGNORECASE, re.IGNORECASE,
) )
if dated_match: if dated_match:
return " ".join(dated_match.group(1).split()) return clean_display_text(dated_match.group(1))
match = re.search(r"(.{0,80}" + re.escape(term) + r".{0,80})", text, re.IGNORECASE) match = re.search(r"(.{0,80}" + re.escape(term) + r".{0,80})", text, re.IGNORECASE)
if match: if match:
return " ".join(match.group(1).split()) return clean_display_text(match.group(1))
return term return clean_display_text(term)
def _find_nearest_link(self, soup: BeautifulSoup, term: str, base_url: str) -> str | None: def _find_nearest_link(self, soup: BeautifulSoup, term: str, base_url: str) -> str | None:
normalized_term = normalize_search_text(term) normalized_term = normalize_search_text(term)