feat: multi-domain failover for provider accounts

Each ProviderAccount can now have multiple base URLs (provider_urls table).
On stream failure, BroadcastGroup cycles to the next domain immediately
with no wait; backs off only after all domains have been tried once.
Background health checker pings every domain every 5 min via player_api.php
and updates status/response_ms. Admin UI shows domain list with color-coded
status badges and a "Verificar todos" button per provider.

backend/core/health_checker.py

@@ -0,0 +1,115 @@
+"""Background health checker for provider URLs."""
+import asyncio
+import logging
+import time
+from datetime import datetime, timezone
+
+import aiohttp
+from sqlalchemy import select
+
+from ..database import AsyncSessionLocal
+from ..models.provider import ProviderAccount, ProviderUrl
+
+logger = logging.getLogger(__name__)
+
+CHECK_INTERVAL = 300  # 5 minutes
+
+
+async def _check_url(base_url: str, username: str, password: str) -> tuple[str, int | None]:
+    url = f"{base_url.rstrip('/')}/player_api.php"
+    params = {"username": username, "password": password, "action": "user_info"}
+    start = time.time()
+    try:
+        timeout = aiohttp.ClientTimeout(total=8)
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.get(url, params=params, ssl=False) as resp:
+                elapsed_ms = int((time.time() - start) * 1000)
+                if resp.status == 200:
+                    try:
+                        data = await resp.json(content_type=None)
+                        if "user_info" in data:
+                            return "ok", elapsed_ms
+                    except Exception:
+                        pass
+                return "error", elapsed_ms
+    except asyncio.TimeoutError:
+        return "timeout", None
+    except Exception:
+        return "error", None
+
+
+async def check_provider_urls(provider_account_id: int) -> list[dict]:
+    """Check all URLs for a specific provider. Returns list of result dicts."""
+    async with AsyncSessionLocal() as db:
+        result = await db.execute(
+            select(ProviderUrl.id, ProviderUrl.url, ProviderAccount.username, ProviderAccount.password)
+            .join(ProviderAccount)
+            .where(
+                ProviderUrl.provider_account_id == provider_account_id,
+                ProviderUrl.is_active == True,  # noqa: E712
+            )
+        )
+        rows = result.all()
+
+    if not rows:
+        return []
+
+    tasks = [_check_url(url, username, password) for _, url, username, password in rows]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+
+    now = datetime.now(timezone.utc)
+    output = []
+    async with AsyncSessionLocal() as db:
+        for (url_id, url, _, _), check_result in zip(rows, results):
+            if isinstance(check_result, Exception):
+                status, response_ms = "error", None
+            else:
+                status, response_ms = check_result
+            pu = await db.get(ProviderUrl, url_id)
+            if pu:
+                pu.status = status
+                pu.response_ms = response_ms
+                pu.last_checked_at = now
+            output.append({"id": url_id, "url": url, "status": status, "response_ms": response_ms})
+        await db.commit()
+
+    return output
+
+
+async def health_check_loop() -> None:
+    """Runs forever, checking all active provider URLs every CHECK_INTERVAL seconds."""
+    logger.info("Provider URL health checker started")
+    while True:
+        await asyncio.sleep(CHECK_INTERVAL)
+        try:
+            async with AsyncSessionLocal() as db:
+                result = await db.execute(
+                    select(ProviderUrl.id, ProviderUrl.url, ProviderAccount.username, ProviderAccount.password)
+                    .join(ProviderAccount)
+                    .where(ProviderUrl.is_active == True)  # noqa: E712
+                )
+                rows = result.all()
+
+            if not rows:
+                continue
+
+            tasks = [_check_url(url, username, password) for _, url, username, password in rows]
+            check_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            now = datetime.now(timezone.utc)
+            async with AsyncSessionLocal() as db:
+                for (url_id, _, _, _), check_result in zip(rows, check_results):
+                    if isinstance(check_result, Exception):
+                        status, response_ms = "error", None
+                    else:
+                        status, response_ms = check_result
+                    pu = await db.get(ProviderUrl, url_id)
+                    if pu:
+                        pu.status = status
+                        pu.response_ms = response_ms
+                        pu.last_checked_at = now
+                await db.commit()
+
+            logger.info(f"Health check done: {len(rows)} URLs checked")
+        except Exception as e:
+            logger.error(f"Health check loop error: {e}")

backend/core/pool.py

@@ -112,9 +112,10 @@ class ProviderPool:
             provider_name = next(
                 (pm.get("provider_name", "") for pm in provider_maps if pm["provider_account_id"] == account_id), ""
             )
+            stream_urls = await self._build_stream_urls(account_id, stream_url)
             group = BroadcastGroup(
                 channel_id=channel_id,
-                stream_url=stream_url,
+                stream_urls=stream_urls,
                 provider_account_id=account_id,
                 channel_name=channel_name,
                 provider_name=provider_name,
@@ -239,6 +240,43 @@ class ProviderPool:
         rest = [pm for pm in provider_maps if pm["provider_account_id"] not in pref_set]
         return preferred + rest
 
+    async def _build_stream_urls(self, account_id: int, primary_stream_url: str) -> list[str]:
+        """Build list of stream URLs for failover: one per active ProviderUrl, substituting base URL."""
+        from urllib.parse import urlparse, urlunparse
+        from ..models.provider import ProviderUrl
+        from ..database import AsyncSessionLocal
+        from sqlalchemy import select
+
+        parsed = urlparse(primary_stream_url)
+
+        async with AsyncSessionLocal() as db:
+            result = await db.execute(
+                select(ProviderUrl)
+                .where(
+                    ProviderUrl.provider_account_id == account_id,
+                    ProviderUrl.is_active == True,  # noqa: E712
+                )
+                .order_by(ProviderUrl.priority)
+            )
+            provider_urls = result.scalars().all()
+
+        if not provider_urls:
+            return [primary_stream_url]
+
+        urls = []
+        for pu in provider_urls:
+            alt = urlparse(pu.url.rstrip('/'))
+            reconstructed = urlunparse((
+                alt.scheme or parsed.scheme,
+                alt.netloc,
+                parsed.path,
+                parsed.params,
+                parsed.query,
+                parsed.fragment,
+            ))
+            urls.append(reconstructed)
+        return urls
+
     async def _find_free_slot(self, provider_maps: list[dict]) -> tuple[int | None, str | None]:
         for pm in provider_maps:
             account_id = pm["provider_account_id"]

backend/core/restream.py

@@ -51,6 +51,14 @@ _SENTINEL = object()   # signals end-of-stream to client queues
 _FFMPEG = shutil.which("ffmpeg") or "ffmpeg"
 
 
+def _url_netloc(url: str) -> str:
+    from urllib.parse import urlparse
+    try:
+        return urlparse(url).netloc
+    except Exception:
+        return url
+
+
 @dataclass
 class ClientHandle:
     client_id: str
@@ -75,14 +83,24 @@ class BroadcastGroup:
     def __init__(
         self,
         channel_id: int,
-        stream_url: str,
+        stream_urls: list[str],
         provider_account_id: int,
         channel_name: str = "",
         provider_name: str = "",
+        # kept for backward compat in callers that still pass stream_url as kwarg
+        stream_url: str | None = None,
     ):
         self.channel_id          = channel_id
         self.channel_name        = channel_name
-        self.stream_url          = stream_url
+        # Support old callers that pass a single stream_url
+        if stream_urls:
+            self._stream_urls = stream_urls
+        elif stream_url:
+            self._stream_urls = [stream_url]
+        else:
+            self._stream_urls = []
+        self.stream_url          = self._stream_urls[0] if self._stream_urls else ""
+        self._active_url: str    = self.stream_url
         self.provider_account_id = provider_account_id
         self.provider_name       = provider_name
         self.started_at          = time.time()
@@ -212,6 +230,9 @@ class BroadcastGroup:
             # ffmpeg process info
             "ffmpeg_pid":       self._ffmpeg_pid,
             "ffmpeg_cpu_pct":   round(self._ffmpeg_cpu, 2),
+            # multi-domain failover info
+            "active_url_domain": _url_netloc(self._active_url),
+            "url_count":         len(self._stream_urls),
         }
 
     # ------------------------------------------------------------------
@@ -256,11 +277,16 @@ class BroadcastGroup:
     async def _pump_with_retry(self) -> None:
         consecutive = 0
         base_wait   = 0.5
+        url_idx     = 0
+        n           = len(self._stream_urls) or 1
 
         try:
             while self._running:
+                current_url = self._stream_urls[url_idx % n]
+                self._active_url = current_url
                 try:
-                    await self._pump_once()
+                    await self._pump_once(current_url)
+                    url_idx = 0  # reset to primary after successful pump
                     break
                 except asyncio.CancelledError:
                     raise
@@ -269,18 +295,31 @@ class BroadcastGroup:
                         break
                     consecutive += 1
                     self._reconnect_count += 1
-                    wait = min(base_wait * (2 ** (consecutive - 1)), _BACKOFF_CAP)
+                    next_idx = (url_idx + 1) % n
+                    cycle_done = (consecutive % n == 0)
+
+                    if n > 1:
+                        logger.warning(
+                            f"[ch{self.channel_id}] Domain {_url_netloc(current_url)} failed "
+                            f"(attempt {consecutive}), trying {_url_netloc(self._stream_urls[next_idx])}: {e}"
+                        )
+                    else:
+                        logger.warning(
+                            f"[ch{self.channel_id}] Upstream error (attempt {consecutive}): {e}"
+                        )
+
+                    url_idx = next_idx
                     self._status = "reconnecting"
-                    logger.warning(
-                        f"[ch{self.channel_id}] ffmpeg exited (attempt {consecutive}), "
-                        f"restarting in {wait:.1f}s: {e}"
-                    )
-                    # Drain queues so stale buffered chunks are not delivered
-                    # after the fresh ffmpeg process starts.
-                    async with self._lock:
-                        for handle in self._clients.values():
-                            _drain_queue(handle.queue)
-                    await asyncio.sleep(wait)
+
+                    if cycle_done:
+                        # All domains tried — drain queues and back off before next cycle
+                        wait = min(base_wait * (2 ** (consecutive // n - 1)), _BACKOFF_CAP)
+                        logger.warning(f"[ch{self.channel_id}] All {n} domain(s) failed, retrying in {wait:.1f}s")
+                        async with self._lock:
+                            for handle in self._clients.values():
+                                _drain_queue(handle.queue)
+                        await asyncio.sleep(wait)
+                    # else: try next domain immediately (no sleep)
         finally:
             self._status  = "stopped"
             self._running = False
@@ -292,7 +331,7 @@ class BroadcastGroup:
                     except asyncio.QueueFull:
                         pass
 
-    async def _pump_once(self) -> None:
+    async def _pump_once(self, url: str | None = None) -> None:
         """
         Spawn ffmpeg in copy (remux) mode and pump its stdout to all client
         queues until ffmpeg exits or self._running goes False.
@@ -302,6 +341,7 @@ class BroadcastGroup:
         to clients.  PTS/DTS values are normalised by ffmpeg across reconnects,
         eliminating A/V desync entirely.
         """
+        stream_url = url if url is not None else self.stream_url
         cmd = [
             _FFMPEG,
             "-hide_banner", "-loglevel", "warning",
@@ -316,7 +356,7 @@ class BroadcastGroup:
             # stops sending bytes for STALL_TIMEOUT seconds ffmpeg closes
             # the connection and tries to reconnect (or exits if it gives up).
             "-timeout", str(int(STALL_TIMEOUT * 1_000_000)),
-            "-i", self.stream_url,
+            "-i", stream_url,
             "-c", "copy",   # remux only — zero transcoding, bit-for-bit quality
             "-f", "mpegts",
             "pipe:1",