diff --git a/.env.example b/.env.example index 03c29ec..2f5abca 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ DB_NAME=eventlens MYSQL_ROOT_PASSWORD=rootsecret TICKETMASTER_API_KEY= -BANDSINTOWN_APP_ID=eventlens +BANDSINTOWN_APP_ID= EVENTIM_ENABLED=true POLL_INTERVAL_HOURS=6 REMINDER_INTERVAL_HOURS=12 diff --git a/README.md b/README.md index 8b58fa6..38e3ff8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ API-Statusinfo findest du unter `http://127.0.0.1:8000/api`. ## Wichtige Umgebungsvariablen - `TICKETMASTER_API_KEY`: Ticketmaster Discovery API -- `BANDSINTOWN_APP_ID`: Bandsintown App-ID fuer Artist-Events +- `BANDSINTOWN_APP_ID`: echte Bandsintown App-ID fuer Artist-Events - `EVENTIM_ENABLED`: aktiviert den Eventim-Website-Provider - `NOTIFICATION_EMAIL_TO`: Empfaenger fuer Benachrichtigungen - `SMTP_HOST`, `SMTP_USER`, `SMTP_PASS`: SMTP-Zugang fuer E-Mails @@ -110,3 +110,7 @@ sudo docker compose up -d --build - Ticketmaster nutzt die offizielle Discovery API. - Bandsintown nutzt die offizielle Artist-Events-API und arbeitet deshalb vor allem fuer Watchlist-Eintraege vom Typ `artist`. - Eventim nutzt eine beobachtete oeffentliche JSON-Web-API von eventim.de. Das ist keine offiziell dokumentierte Developer-API, aber stabiler als HTML-Scraping. Aenderungen an diesem Web-Endpoint koennen trotzdem Anpassungen noetig machen. +- Eventim kann serverseitig durch Eventim/Akamai geblockt werden. In dem Fall liefert `eventlens` bewusst keine falschen Treffer, sondern ueberspringt den Provider und schreibt einen Hinweis ins Backend-Log. +- Bandsintown benoetigt eine echte, von Bandsintown freigeschaltete App-ID. Ohne diese wird der Provider deaktiviert oder als `blocked` angezeigt. +- Barclays Arena wird ueber die offizielle Eventseite der Arena abgefragt. +- Fabrik wird ueber die offizielle Veranstaltungsseite der Fabrik Hamburg abgefragt. diff --git a/backend/app/config.py b/backend/app/config.py index 018590d..2ce2a00 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -12,7 +12,7 @@ class Settings: db_name = os.getenv("DB_NAME", "eventlens") ticketmaster_api_key = os.getenv("TICKETMASTER_API_KEY", "") - bandsintown_app_id = os.getenv("BANDSINTOWN_APP_ID", "eventlens") + bandsintown_app_id = os.getenv("BANDSINTOWN_APP_ID", "") eventim_enabled = os.getenv("EVENTIM_ENABLED", "true").lower() == "true" poll_interval_hours = int(os.getenv("POLL_INTERVAL_HOURS", "6")) reminder_interval_hours = int(os.getenv("REMINDER_INTERVAL_HOURS", "12")) diff --git a/backend/app/frontend/index.html b/backend/app/frontend/index.html index 63a60ef..492e8e0 100644 --- a/backend/app/frontend/index.html +++ b/backend/app/frontend/index.html @@ -115,6 +115,8 @@ + + @@ -124,6 +126,19 @@ +
+
+
+

Provider

+

Status der Quellen

+
+
+ +
+ Noch keine Provider-Statusdaten vorhanden. +
+
+
diff --git a/backend/app/frontend/static/app.js b/backend/app/frontend/static/app.js index b065306..2cb76a2 100644 --- a/backend/app/frontend/static/app.js +++ b/backend/app/frontend/static/app.js @@ -2,6 +2,7 @@ const state = { watchItems: [], events: [], notifications: [], + providerStatuses: [], }; const watchItemsEl = document.querySelector("#watch-items"); @@ -13,6 +14,7 @@ const providerFilter = document.querySelector("#provider-filter"); const syncButton = document.querySelector("#sync-button"); const toastEl = document.querySelector("#toast"); const providerSummaryEl = document.querySelector("#provider-summary"); +const providerStatusListEl = document.querySelector("#provider-status-list"); function showToast(message) { toastEl.textContent = message; @@ -28,7 +30,12 @@ function formatDate(value) { return "unbekannt"; } - const date = new Date(value); + const normalizedValue = + typeof value === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?$/.test(value) + ? `${value}Z` + : value; + + const date = new Date(normalizedValue); if (Number.isNaN(date.getTime())) { return value; } @@ -89,6 +96,8 @@ function prettifyProviderName(value) { ticketmaster: "Ticketmaster", bandsintown: "Bandsintown", eventim: "Eventim", + barclays_arena: "Barclays Arena", + fabrik: "Fabrik", }; return names[value] || value; @@ -99,6 +108,8 @@ function getProviderClass(value) { ticketmaster: "provider-ticketmaster", bandsintown: "provider-bandsintown", eventim: "provider-eventim", + barclays_arena: "provider-barclays-arena", + fabrik: "provider-fabrik", }; return classes[value] || ""; @@ -128,6 +139,40 @@ function renderProviderSummary() { .join(""); } +function renderProviderStatuses() { + if (!state.providerStatuses.length) { + providerStatusListEl.className = "provider-status-list empty-state"; + providerStatusListEl.textContent = "Noch keine Provider-Statusdaten vorhanden."; + return; + } + + providerStatusListEl.className = "provider-status-list"; + providerStatusListEl.innerHTML = state.providerStatuses + .map( + (entry) => ` +
+
+
+

${escapeHtml(prettifyProviderName(entry.provider_name))}

+
+ ${escapeHtml(entry.status)} +
+
+ ${escapeHtml(formatDate(entry.last_checked_at))} +
+

${escapeHtml(entry.message)}

+
+ ` + ) + .join(""); +} + function renderWatchItems() { if (!state.watchItems.length) { watchItemsEl.className = "watch-list empty-state"; @@ -283,19 +328,22 @@ function updateSyncStatus(message) { } async function loadData() { - const [watchItems, events, notifications] = await Promise.all([ + const [watchItems, events, notifications, providerStatuses] = await Promise.all([ apiFetch("/watch-items"), apiFetch("/events"), apiFetch("/notifications"), + apiFetch("/provider-statuses"), ]); state.watchItems = watchItems; state.events = events; state.notifications = notifications; + state.providerStatuses = providerStatuses; renderStats(); renderWatchItems(); renderEvents(); + renderProviderStatuses(); renderNotifications(); } diff --git a/backend/app/frontend/static/styles.css b/backend/app/frontend/static/styles.css index 9fcccfb..86833a2 100644 --- a/backend/app/frontend/static/styles.css +++ b/backend/app/frontend/static/styles.css @@ -255,7 +255,8 @@ button:hover, .watch-list, .event-list, -.notification-list { +.notification-list, +.provider-status-list { display: grid; gap: 14px; } @@ -270,7 +271,8 @@ button:hover, .watch-card, .event-card, -.notification-card { +.notification-card, +.provider-status-card { padding: 18px; border-radius: var(--radius-md); background: var(--surface-strong); @@ -357,13 +359,24 @@ button:hover, color: #4f3790; } +.provider-barclays-arena { + background: rgba(26, 87, 163, 0.14); + color: #1a57a3; +} + +.provider-fabrik { + background: rgba(154, 77, 24, 0.14); + color: #8e4516; +} + .muted { color: var(--muted); } .watch-card p, .event-card p, -.notification-card p { +.notification-card p, +.provider-status-card p { margin-top: 12px; line-height: 1.55; } diff --git a/backend/app/main.py b/backend/app/main.py index 695778c..ade1aab 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -12,6 +12,7 @@ from app.models import TrackedEvent, WatchItem from app.scheduler import start_scheduler from app.schemas import ( NotificationLogRead, + ProviderStatusRead, PurchaseUpdate, SyncResult, TrackedEventRead, @@ -19,7 +20,13 @@ from app.schemas import ( WatchItemRead, WatchItemUpdate, ) -from app.services import list_events, list_notifications, list_watch_items, run_sync +from app.services import ( + list_events, + list_notifications, + list_provider_statuses, + list_watch_items, + run_sync, +) app = FastAPI( @@ -135,6 +142,11 @@ def get_notifications(db: Session = Depends(get_db)): return list_notifications(db) +@app.get("/provider-statuses", response_model=list[ProviderStatusRead]) +def get_provider_statuses(db: Session = Depends(get_db)): + return list_provider_statuses(db) + + @app.post("/sync", response_model=SyncResult) def trigger_sync(db: Session = Depends(get_db)): return run_sync(db) diff --git a/backend/app/models.py b/backend/app/models.py index f15a100..03a7bcc 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -29,6 +29,12 @@ class NotificationStatus(str, Enum): failed = "failed" +class ProviderStatusType(str, Enum): + ok = "ok" + blocked = "blocked" + error = "error" + + class WatchItem(Base): __tablename__ = "watch_items" @@ -109,3 +115,17 @@ class NotificationLog(Base): tracked_event: Mapped[TrackedEvent] = relationship(back_populates="notifications") + +class ProviderStatus(Base): + __tablename__ = "provider_statuses" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + provider_name: Mapped[str] = mapped_column(String(50), unique=True, nullable=False) + status: Mapped[ProviderStatusType] = mapped_column( + SqlEnum(ProviderStatusType), nullable=False + ) + message: Mapped[str] = mapped_column(Text, nullable=False) + last_checked_at: Mapped[datetime] = mapped_column( + DateTime, default=datetime.utcnow, nullable=False + ) + last_success_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) diff --git a/backend/app/providers/bandsintown.py b/backend/app/providers/bandsintown.py index 7bd3ebc..0c85192 100644 --- a/backend/app/providers/bandsintown.py +++ b/backend/app/providers/bandsintown.py @@ -5,6 +5,7 @@ import requests from app.config import settings from app.models import RegionScope, WatchType +from app.providers.utils import normalize_search_text class BandsintownProvider: @@ -21,9 +22,21 @@ class BandsintownProvider: region_scope: RegionScope, ) -> list[dict]: if not self.is_configured(): + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + "Bandsintown ist nicht konfiguriert. Bitte eine gueltige BANDSINTOWN_APP_ID setzen.", + ) return [] if watch_type != WatchType.artist: + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + f"Bandsintown skipped non-artist watch item '{term}'.", + ) return [] artist_name = quote(term, safe="") @@ -34,15 +47,46 @@ class BandsintownProvider: timeout=30, ) + if response.status_code in {401, 403, 429}: + setattr(self, "last_status", "blocked") + setattr( + self, + "last_message", + ( + f"Bandsintown rejected app_id for '{term}' " + f"with status {response.status_code}." + ), + ) + return [] + if response.status_code == 404: + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + f"Bandsintown found no artist match for '{term}'.", + ) return [] response.raise_for_status() payload = response.json() if not isinstance(payload, list): + setattr(self, "last_status", "error") + setattr( + self, + "last_message", + f"Bandsintown returned an unexpected payload for '{term}'.", + ) return [] + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + f"Bandsintown returned {len(payload)} raw events for term '{term}'.", + ) results: list[dict] = [] + normalized_term = normalize_search_text(term) for item in payload: venue = item.get("venue") or {} city = venue.get("city") @@ -66,10 +110,16 @@ class BandsintownProvider: offers = item.get("offers") or [] ticket_url = next((offer.get("url") for offer in offers if offer.get("url")), None) + title = item.get("title") or f"{term} live" + artist_name = item.get("artist", {}).get("name", term) + haystack = normalize_search_text(f"{artist_name} {title}") + if normalized_term not in haystack: + continue + results.append( { "external_id": str(item.get("id")), - "title": item.get("title") or f"{term} live", + "title": title, "matched_term": term, "venue_name": venue.get("name"), "city": city, @@ -82,4 +132,3 @@ class BandsintownProvider: ) return results - diff --git a/backend/app/providers/barclays_arena.py b/backend/app/providers/barclays_arena.py new file mode 100644 index 0000000..bcdf841 --- /dev/null +++ b/backend/app/providers/barclays_arena.py @@ -0,0 +1,101 @@ +from datetime import datetime +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup + +from app.models import RegionScope, WatchType +from app.providers.utils import normalize_search_text + + +class BarclaysArenaProvider: + source_name = "barclays_arena" + events_url = "https://www.barclays-arena.de/events" + + def search_events( + self, + term: str, + watch_type: WatchType, + region_scope: RegionScope, + ) -> list[dict]: + response = requests.get( + self.events_url, + headers={"User-Agent": "Mozilla/5.0"}, + timeout=30, + ) + response.raise_for_status() + soup = BeautifulSoup(response.text, "html.parser") + + normalized_term = normalize_search_text(term) + results: list[dict] = [] + + headings = soup.find_all("h3") + for heading in headings: + title = heading.get_text(" ", strip=True) + if not title: + continue + + subtitle = "" + subtitle_el = heading.find_next("h4") + if subtitle_el: + subtitle = subtitle_el.get_text(" ", strip=True) + + haystack = normalize_search_text(f"{title} {subtitle}") + if normalized_term not in haystack: + continue + + date_text = self._find_previous_date_text(heading) + event_date = self._parse_german_date(date_text) + + link = heading.find_previous("a", href=True) + if link is None: + continue + + results.append( + { + "external_id": link["href"], + "title": title, + "matched_term": term, + "venue_name": "Barclays Arena", + "city": "Hamburg", + "country_code": "DE", + "event_date": event_date, + "ticket_url": urljoin(self.events_url, link["href"]), + "image_url": None, + "raw_payload": { + "title": title, + "subtitle": subtitle, + "date_text": date_text, + "href": link["href"], + }, + } + ) + + self.last_status = "ok" + self.last_message = ( + f"Barclays Arena returned {len(results)} matched events for term '{term}'." + ) + return results + + def _find_previous_date_text(self, heading) -> str | None: + current = heading.previous_sibling + while current is not None: + text = getattr(current, "get_text", lambda *args, **kwargs: str(current))( + " ", strip=True + ) + if text and "|" in text: + return text + current = current.previous_sibling + return None + + def _parse_german_date(self, value: str | None) -> datetime | None: + if not value: + return None + parts = [part.strip() for part in value.split("|")] + if len(parts) < 2: + return None + try: + return datetime.strptime(parts[1], "%d.%m.%Y") + except ValueError: + return None + diff --git a/backend/app/providers/eventim.py b/backend/app/providers/eventim.py index 6715368..2d368af 100644 --- a/backend/app/providers/eventim.py +++ b/backend/app/providers/eventim.py @@ -1,9 +1,14 @@ from datetime import datetime +import logging import requests from app.config import settings from app.models import RegionScope, WatchType +from app.providers.utils import normalize_search_text + + +logger = logging.getLogger(__name__) class EventimProvider: @@ -46,12 +51,31 @@ class EventimProvider: }, timeout=30, ) + if response.status_code in {403, 429}: + logger.warning( + "Eventim API blocked request with status %s for term '%s'.", + response.status_code, + term, + ) + setattr(self, "last_status", "blocked") + setattr( + self, + "last_message", + f"Eventim API blocked request with status {response.status_code} for term '{term}'.", + ) + return [] response.raise_for_status() payload = response.json() products = payload.get("products") or [] + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + f"Eventim returned {len(products)} raw products for term '{term}'.", + ) results: list[dict] = [] - normalized_term = term.casefold() + normalized_term = normalize_search_text(term) for product in products: title = product.get("name") or "" @@ -59,10 +83,10 @@ class EventimProvider: attraction_names = [entry.get("name", "") for entry in attractions] if watch_type == WatchType.artist: - haystack = " ".join(attraction_names + [title]).casefold() + haystack = normalize_search_text(" ".join(attraction_names + [title])) if normalized_term not in haystack: continue - elif normalized_term not in title.casefold(): + elif normalized_term not in normalize_search_text(title): continue live_data = product.get("typeAttributes", {}).get("liveEntertainment", {}) diff --git a/backend/app/providers/fabrik.py b/backend/app/providers/fabrik.py new file mode 100644 index 0000000..fa288e5 --- /dev/null +++ b/backend/app/providers/fabrik.py @@ -0,0 +1,88 @@ +from datetime import datetime +from urllib.parse import urljoin + +import requests +from bs4 import BeautifulSoup + +from app.models import RegionScope, WatchType +from app.providers.utils import normalize_search_text + + +class FabrikProvider: + source_name = "fabrik" + events_url = "https://fabrik.de/programm" + + def search_events( + self, + term: str, + watch_type: WatchType, + region_scope: RegionScope, + ) -> list[dict]: + response = requests.get( + self.events_url, + headers={"User-Agent": "Mozilla/5.0"}, + timeout=30, + ) + response.raise_for_status() + soup = BeautifulSoup(response.text, "html.parser") + + normalized_term = normalize_search_text(term) + results: list[dict] = [] + + for event_block in soup.select(".programm_termin"): + detail_link = event_block.select_one('a[href*="/veranstaltungsdetail/"][title]') + if detail_link is None: + continue + + title = detail_link.get("title") or detail_link.get_text(" ", strip=True) + if not title: + continue + + block_text = event_block.get_text(" ", strip=True) + haystack = normalize_search_text(f"{title} {block_text}") + if normalized_term not in haystack: + continue + + date_el = event_block.select_one(".programm_termin_lh_datum a") + date_text = date_el.get_text(" ", strip=True) if date_el else "" + event_date = self._parse_fabrik_date(date_text) + href = detail_link["href"] + + results.append( + { + "external_id": href, + "title": title, + "matched_term": term, + "venue_name": "Fabrik", + "city": "Hamburg", + "country_code": "DE", + "event_date": event_date, + "ticket_url": urljoin(self.events_url, href), + "image_url": None, + "raw_payload": { + "title": title, + "date_text": date_text, + "href": href, + }, + } + ) + + unique_results: dict[str, dict] = {} + for result in results: + unique_results[result["external_id"]] = result + + self.last_status = "ok" + self.last_message = ( + f"Fabrik returned {len(unique_results)} matched events for term '{term}'." + ) + return list(unique_results.values()) + + def _parse_fabrik_date(self, value: str) -> datetime | None: + cleaned = value.replace(" ", "") + if len(cleaned) < 10: + return None + date_value = cleaned[-10:] + try: + return datetime.strptime(date_value, "%d.%m.%Y") + except ValueError: + return None diff --git a/backend/app/providers/registry.py b/backend/app/providers/registry.py index 685bf9a..93c6a5e 100644 --- a/backend/app/providers/registry.py +++ b/backend/app/providers/registry.py @@ -1,5 +1,7 @@ from app.providers.bandsintown import BandsintownProvider +from app.providers.barclays_arena import BarclaysArenaProvider from app.providers.eventim import EventimProvider +from app.providers.fabrik import FabrikProvider from app.providers.ticketmaster import TicketmasterProvider @@ -8,5 +10,6 @@ def get_providers(): TicketmasterProvider(), BandsintownProvider(), EventimProvider(), + BarclaysArenaProvider(), + FabrikProvider(), ] - diff --git a/backend/app/providers/ticketmaster.py b/backend/app/providers/ticketmaster.py index bdfe696..d2a7cad 100644 --- a/backend/app/providers/ticketmaster.py +++ b/backend/app/providers/ticketmaster.py @@ -4,6 +4,7 @@ import requests from app.config import settings from app.models import RegionScope, WatchType +from app.providers.utils import normalize_search_text class TicketmasterProvider: @@ -38,19 +39,25 @@ class TicketmasterProvider: response.raise_for_status() payload = response.json() items = payload.get("_embedded", {}).get("events", []) + setattr(self, "last_status", "ok") + setattr( + self, + "last_message", + f"Ticketmaster returned {len(items)} raw events for term '{term}'.", + ) results: list[dict] = [] - normalized_term = term.casefold() + normalized_term = normalize_search_text(term) for item in items: title = item.get("name", "") attractions = item.get("_embedded", {}).get("attractions", []) attraction_names = [entry.get("name", "") for entry in attractions] if watch_type == WatchType.artist: - haystack = " ".join(attraction_names + [title]).casefold() + haystack = normalize_search_text(" ".join(attraction_names + [title])) if normalized_term not in haystack: continue - elif normalized_term not in title.casefold(): + elif normalized_term not in normalize_search_text(title): continue dates = item.get("dates", {}).get("start", {}) @@ -88,4 +95,3 @@ class TicketmasterProvider: ) return results - diff --git a/backend/app/providers/utils.py b/backend/app/providers/utils.py new file mode 100644 index 0000000..042d78d --- /dev/null +++ b/backend/app/providers/utils.py @@ -0,0 +1,16 @@ +import unicodedata + + +def normalize_search_text(value: str | None) -> str: + if not value: + return "" + + value = ( + value.casefold() + .replace("ä", "ae") + .replace("ö", "oe") + .replace("ü", "ue") + .replace("ß", "ss") + ) + normalized = unicodedata.normalize("NFKD", value) + return "".join(char for char in normalized if not unicodedata.combining(char)) diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 90423c3..611efb4 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -2,7 +2,13 @@ from datetime import datetime from pydantic import BaseModel, ConfigDict, Field -from app.models import NotificationStatus, NotificationType, RegionScope, WatchType +from app.models import ( + NotificationStatus, + NotificationType, + ProviderStatusType, + RegionScope, + WatchType, +) class WatchItemCreate(BaseModel): @@ -78,3 +84,13 @@ class SyncResult(BaseModel): notifications_sent: int notifications_skipped: int + +class ProviderStatusRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + provider_name: str + status: ProviderStatusType + message: str + last_checked_at: datetime + last_success_at: datetime | None diff --git a/backend/app/services.py b/backend/app/services.py index fd09663..85cc90b 100644 --- a/backend/app/services.py +++ b/backend/app/services.py @@ -1,21 +1,38 @@ from datetime import datetime, timedelta +import logging import re from sqlalchemy import desc, select from sqlalchemy.orm import Session -from app.models import NotificationStatus, NotificationType, TrackedEvent, WatchItem +from app.models import ( + NotificationStatus, + NotificationType, + ProviderStatus, + ProviderStatusType, + TrackedEvent, + WatchItem, +) from app.notifications import send_email_notification from app.providers.registry import get_providers from app.schemas import SyncResult +logger = logging.getLogger(__name__) + + PROVIDER_PRIORITY = { "ticketmaster": 3, "eventim": 2, "bandsintown": 1, } +PROVIDER_STATUS_PRIORITY = { + ProviderStatusType.error: 3, + ProviderStatusType.blocked: 2, + ProviderStatusType.ok: 1, +} + def list_watch_items(db: Session) -> list[WatchItem]: return list(db.scalars(select(WatchItem).order_by(WatchItem.name))) @@ -50,6 +67,86 @@ def list_notifications(db: Session): return list(db.scalars(select(NotificationLog).order_by(desc(NotificationLog.created_at)))) +def list_provider_statuses(db: Session) -> list[ProviderStatus]: + return list(db.scalars(select(ProviderStatus).order_by(ProviderStatus.provider_name))) + + +def update_provider_status( + db: Session, + provider_name: str, + status: ProviderStatusType, + message: str, +): + provider_status = db.scalar( + select(ProviderStatus).where(ProviderStatus.provider_name == provider_name) + ) + now = datetime.utcnow() + + if provider_status is None: + provider_status = ProviderStatus( + provider_name=provider_name, + status=status, + message=message, + last_checked_at=now, + last_success_at=now if status == ProviderStatusType.ok else None, + ) + db.add(provider_status) + return + + provider_status.status = status + provider_status.message = message + provider_status.last_checked_at = now + if status == ProviderStatusType.ok: + provider_status.last_success_at = now + + +def init_provider_sync_state(provider_name: str) -> dict: + return { + "provider_name": provider_name, + "status": ProviderStatusType.ok, + "ok_count": 0, + "blocked_terms": [], + "error_terms": [], + "last_message": "", + } + + +def record_provider_sync_state( + state: dict, + status: ProviderStatusType, + term: str, + message: str, +): + if PROVIDER_STATUS_PRIORITY[status] > PROVIDER_STATUS_PRIORITY[state["status"]]: + state["status"] = status + + if status == ProviderStatusType.ok: + state["ok_count"] += 1 + elif status == ProviderStatusType.blocked: + if term not in state["blocked_terms"]: + state["blocked_terms"].append(term) + elif status == ProviderStatusType.error: + if term not in state["error_terms"]: + state["error_terms"].append(term) + + state["last_message"] = message + + +def build_provider_status_message(state: dict) -> str: + if state["status"] == ProviderStatusType.error: + terms = ", ".join(state["error_terms"][:3]) + return f"Fehler bei {state['provider_name']} fuer: {terms}." + + if state["status"] == ProviderStatusType.blocked: + terms = ", ".join(state["blocked_terms"][:3]) + return f"{state['provider_name']} wurde geblockt fuer: {terms}." + + return ( + f"{state['provider_name']} erfolgreich fuer {state['ok_count']} " + f"Watchlist-Eintraege geprueft." + ) + + def normalize_event_text(value: str | None) -> str: if not value: return "" @@ -164,6 +261,10 @@ def upsert_event( def run_sync(db: Session) -> SyncResult: providers = get_providers() + provider_states = { + provider.source_name: init_provider_sync_state(provider.source_name) + for provider in providers + } active_items = list( db.scalars(select(WatchItem).where(WatchItem.is_active.is_(True)).order_by(WatchItem.name)) ) @@ -181,8 +282,31 @@ def run_sync(db: Session) -> SyncResult: watch_type=watch_item.watch_type, region_scope=watch_item.region_scope, ) + provider_status_name = getattr(provider, "last_status", "ok") + provider_message = getattr( + provider, + "last_message", + f"{provider.source_name} completed successfully for '{watch_item.name}'.", + ) + record_provider_sync_state( + state=provider_states[provider.source_name], + status=ProviderStatusType(provider_status_name), + term=watch_item.name, + message=provider_message, + ) except Exception: + logger.exception( + "Provider sync failed for provider=%s watch_item=%s", + provider.source_name, + watch_item.name, + ) db.rollback() + record_provider_sync_state( + state=provider_states[provider.source_name], + status=ProviderStatusType.error, + term=watch_item.name, + message=f"Provider sync failed for '{watch_item.name}'.", + ) continue for event_data in events: tracked_event, is_new = upsert_event( @@ -225,6 +349,15 @@ def run_sync(db: Session) -> SyncResult: notifications_skipped += 1 db.commit() + for provider_name, state in provider_states.items(): + update_provider_status( + db=db, + provider_name=provider_name, + status=state["status"], + message=build_provider_status_message(state), + ) + db.commit() + return SyncResult( scanned_watch_items=len(active_items), new_events=new_events, diff --git a/backend/requirements.txt b/backend/requirements.txt index 5f147ae..de907ba 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,8 +1,8 @@ apscheduler==3.10.4 +beautifulsoup4==4.12.3 fastapi==0.115.12 pymysql==1.1.1 python-dotenv==1.0.1 requests==2.32.3 sqlalchemy==2.0.40 uvicorn[standard]==0.34.1 -