fix: harden Remnawave panel timeouts

backend/bot/app/web/admin_settings_manifest.py

@@ -101,6 +101,42 @@ SETTINGS_MANIFEST: List[SettingField] = [
         "Секретный ключ API панели.",
         secret=True,
     ),
+    SettingField(
+        "PANEL_API_TOTAL_TIMEOUT_SECONDS",
+        "float",
+        "remnawave",
+        "Panel API total timeout",
+        "Maximum total time for one Remnawave API request, in seconds.",
+        optional=False,
+        min=1,
+    ),
+    SettingField(
+        "PANEL_API_CONNECT_TIMEOUT_SECONDS",
+        "float",
+        "remnawave",
+        "Panel API connect timeout",
+        "Maximum time to get or open a Remnawave API connection, in seconds.",
+        optional=False,
+        min=1,
+    ),
+    SettingField(
+        "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
+        "float",
+        "remnawave",
+        "Panel API socket connect timeout",
+        "Maximum TCP/TLS connection time for Remnawave API, in seconds.",
+        optional=False,
+        min=1,
+    ),
+    SettingField(
+        "PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
+        "float",
+        "remnawave",
+        "Panel API socket read timeout",
+        "Maximum time to wait for response data from Remnawave API, in seconds.",
+        optional=False,
+        min=1,
+    ),
     SettingField(
         "PANEL_WEBHOOK_SECRET",
         "string",

backend/bot/services/panel_api_service.py

@@ -22,6 +22,11 @@ class PanelApiService:
     _TRANSIENT_STATUS_CODES = (-1, -3)
     _SAFE_METHODS = frozenset({"GET", "HEAD"})
     _RETRY_BACKOFF_SECONDS = 0.5
+    _MIN_TIMEOUT_SECONDS = 0.1
+    _DEFAULT_TOTAL_TIMEOUT_SECONDS = 25.0
+    _DEFAULT_CONNECT_TIMEOUT_SECONDS = 8.0
+    _DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS = 8.0
+    _DEFAULT_SOCK_READ_TIMEOUT_SECONDS = 15.0
 
     def __init__(self, settings: Settings):
         self.settings = settings
@@ -70,17 +75,46 @@ class PanelApiService:
 
     async def _get_session(self) -> aiohttp.ClientSession:
         if self._session is None or self._session.closed:
-            # Separate connect/read timeouts so a stuck panel does not hold a
+            self._session = aiohttp.ClientSession(timeout=self._client_timeout())
-            # bot worker for the full window; total caps worst-case latency.
-            timeout = aiohttp.ClientTimeout(
-                total=15,
-                connect=3,
-                sock_connect=3,
-                sock_read=10,
-            )
-            self._session = aiohttp.ClientSession(timeout=timeout)
         return self._session
 
+    @classmethod
+    def _timeout_setting(cls, settings: Settings, name: str, default: float) -> float:
+        raw_value = getattr(settings, name, default)
+        try:
+            value = float(raw_value)
+        except (TypeError, ValueError):
+            return default
+        if value <= 0:
+            return default
+        return max(cls._MIN_TIMEOUT_SECONDS, value)
+
+    def _client_timeout(self) -> aiohttp.ClientTimeout:
+        # Separate connect/read timeouts so a slow panel route has more room,
+        # while genuinely stuck requests still cannot pin a worker forever.
+        return aiohttp.ClientTimeout(
+            total=self._timeout_setting(
+                self.settings,
+                "PANEL_API_TOTAL_TIMEOUT_SECONDS",
+                self._DEFAULT_TOTAL_TIMEOUT_SECONDS,
+            ),
+            connect=self._timeout_setting(
+                self.settings,
+                "PANEL_API_CONNECT_TIMEOUT_SECONDS",
+                self._DEFAULT_CONNECT_TIMEOUT_SECONDS,
+            ),
+            sock_connect=self._timeout_setting(
+                self.settings,
+                "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
+                self._DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS,
+            ),
+            sock_read=self._timeout_setting(
+                self.settings,
+                "PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
+                self._DEFAULT_SOCK_READ_TIMEOUT_SECONDS,
+            ),
+        )
+
     async def close_session(self):
         if self._session and not self._session.closed:
             await self._session.close()
@@ -121,6 +155,15 @@ class PanelApiService:
         for attempt in range(max_attempts):
             result = await self._request_once(method, endpoint, log_full_response, **kwargs)
             if attempt + 1 < max_attempts and self._is_transient_error(result):
+                logging.warning(
+                    "Retrying transient Panel API request method=%s endpoint=%s "
+                    "attempt=%s/%s status_code=%s",
+                    method.upper(),
+                    endpoint,
+                    attempt + 1,
+                    max_attempts,
+                    result.get("status_code") if isinstance(result, dict) else None,
+                )
                 await asyncio.sleep(self._RETRY_BACKOFF_SECONDS)
                 continue
             return result
@@ -158,8 +201,8 @@ class PanelApiService:
                 )
             except Exception:
                 log_prefix += f" | Payload: {str(json_payload_for_log)[:300]}..."
+        started = time.monotonic()
         try:
-            started = time.monotonic()
             async with aiohttp_session.request(
                 method.upper(), url_for_request, headers=headers, **kwargs
             ) as response:
@@ -228,15 +271,48 @@ class PanelApiService:
                     return {"error": True, "status_code": response_status, "details": error_details}
 
         except aiohttp.ClientConnectorError as e:
+            logging.info(
+                "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=connect_error",
+                time.monotonic() - started,
+                method.upper(),
+                endpoint,
+            )
             logging.error(f"Panel API ClientConnectorError to {url_for_request}: {e}")
             return {"error": True, "status_code": -1, "message": f"Connection error: {str(e)}"}
+        except aiohttp.ServerTimeoutError as e:
+            logging.info(
+                "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout",
+                time.monotonic() - started,
+                method.upper(),
+                endpoint,
+            )
+            logging.warning("Panel API timeout to %s: %s", url_for_request, e)
+            return {"error": True, "status_code": -3, "message": f"Request timed out: {str(e)}"}
         except aiohttp.ClientError as e:
+            logging.info(
+                "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=client_error",
+                time.monotonic() - started,
+                method.upper(),
+                endpoint,
+            )
             logging.exception("Panel API ClientError to %s.", url_for_request)
             return {"error": True, "status_code": -2, "message": f"Client error: {str(e)}"}
         except asyncio.TimeoutError:
+            logging.info(
+                "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout",
+                time.monotonic() - started,
+                method.upper(),
+                endpoint,
+            )
             logging.error(f"Panel API request to {url_for_request} timed out.")
             return {"error": True, "status_code": -3, "message": "Request timed out"}
         except Exception as e:
+            logging.info(
+                "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=unexpected_error",
+                time.monotonic() - started,
+                method.upper(),
+                endpoint,
+            )
             logging.error(
                 f"Unexpected Panel API request error to {url_for_request}: {e}", exc_info=True
             )
@@ -885,7 +961,14 @@ class PanelApiService:
         await self._devices_cache.invalidate_remote(f"user:{user_uuid}")
 
     async def get_internal_squads(self) -> Optional[List[Dict[str, Any]]]:
-        return await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached)
+        squads = await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached)
+        if squads is not None:
+            return squads
+        stale_squads = self._squads_cache.get_stale("list")
+        if stale_squads is not None:
+            logging.warning("Using stale internal squads cache after panel fetch failed.")
+            return stale_squads
+        return None
 
     async def _get_internal_squads_uncached(self) -> Optional[List[Dict[str, Any]]]:
         response_data = await self._request("GET", "/internal-squads", log_full_response=False)

backend/bot/utils/ttl_cache.py

@@ -29,6 +29,15 @@ class AsyncTTLCache:
             return None
         return value
 
+    def get_stale(self, key: str) -> Optional[Any]:
+        entry = self._data.get(key)
+        if entry is None:
+            return None
+        _, value = entry
+        if not self._is_cacheable(value):
+            return None
+        return value
+
     @staticmethod
     def _is_cacheable(value: Any) -> bool:
         if value is None:

backend/config/settings.py

@@ -100,6 +100,10 @@ class Settings(BaseSettings):
     PANEL_DEVICES_CACHE_TTL_SECONDS: int = Field(default=5)
     PANEL_ALL_USERS_CACHE_TTL_SECONDS: int = Field(default=5)
     PANEL_ALL_USERS_PAGE_SIZE: int = Field(default=1000)
+    PANEL_API_TOTAL_TIMEOUT_SECONDS: float = Field(default=25)
+    PANEL_API_CONNECT_TIMEOUT_SECONDS: float = Field(default=8)
+    PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS: float = Field(default=8)
+    PANEL_API_SOCK_READ_TIMEOUT_SECONDS: float = Field(default=15)
     ADMIN_PANEL_STATS_CACHE_TTL_SECONDS: int = Field(default=15)
     ADMIN_DB_STATS_CACHE_TTL_SECONDS: int = Field(default=5)
     ADMIN_USERS_LIST_CACHE_TTL_SECONDS: int = Field(default=3)

docs/configuration/env-vars.md

@@ -59,6 +59,10 @@
 | `PANEL_DEVICES_CACHE_TTL_SECONDS` | TTL кеша устройств пользователя Remnawave. |
 | `PANEL_ALL_USERS_CACHE_TTL_SECONDS` | TTL кеша полных сканов пользователей Remnawave. |
 | `PANEL_ALL_USERS_PAGE_SIZE` | Размер страницы Remnawave `/users`. |
+| `PANEL_API_TOTAL_TIMEOUT_SECONDS` | Общий timeout запроса к Remnawave API. |
+| `PANEL_API_CONNECT_TIMEOUT_SECONDS` | Timeout получения соединения с Remnawave API. |
+| `PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS` | Timeout TCP/TLS-подключения к Remnawave API. |
+| `PANEL_API_SOCK_READ_TIMEOUT_SECONDS` | Timeout ожидания данных ответа Remnawave API. |
 | `ADMIN_PANEL_STATS_CACHE_TTL_SECONDS` | TTL статистики Remnawave в админке. |
 | `ADMIN_DB_STATS_CACHE_TTL_SECONDS` | TTL дорогих DB-агрегатов админки. |
 | `ADMIN_USERS_LIST_CACHE_TTL_SECONDS` | TTL списка пользователей админки. |

locales/en.json

@@ -1544,6 +1544,14 @@
   "admin_settings_field_panel_api_url_description": "For example, https://panel.example.com/api.",
   "admin_settings_field_panel_api_key_label": "Remnawave API key",
   "admin_settings_field_panel_api_key_description": "Secret API key for the panel.",
+  "admin_settings_field_panel_api_total_timeout_seconds_label": "Remnawave API total timeout",
+  "admin_settings_field_panel_api_total_timeout_seconds_description": "Maximum total time for one Remnawave API request, in seconds.",
+  "admin_settings_field_panel_api_connect_timeout_seconds_label": "Remnawave API connect timeout",
+  "admin_settings_field_panel_api_connect_timeout_seconds_description": "Maximum time to get or open a Remnawave API connection, in seconds.",
+  "admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "Remnawave API TCP/TLS timeout",
+  "admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Maximum TCP/TLS connection time for Remnawave API, in seconds.",
+  "admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Remnawave API read timeout",
+  "admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Maximum time to wait for response data from Remnawave API, in seconds.",
   "admin_settings_field_panel_webhook_secret_label": "Remnawave webhook secret",
   "admin_settings_field_panel_webhook_secret_description": "Set the secret in Remnawave Panel and paste the same value here to verify incoming panel webhooks.",
   "admin_settings_field_user_squad_uuids_label": "Default Internal Squads",

locales/ru.json

@@ -1544,6 +1544,14 @@
   "admin_settings_field_panel_api_url_description": "Например, https://panel.example.com/api.",
   "admin_settings_field_panel_api_key_label": "API-ключ Remnawave",
   "admin_settings_field_panel_api_key_description": "Секретный ключ API панели.",
+  "admin_settings_field_panel_api_total_timeout_seconds_label": "Общий таймаут API Remnawave",
+  "admin_settings_field_panel_api_total_timeout_seconds_description": "Максимальное время одного запроса к Remnawave API, в секундах.",
+  "admin_settings_field_panel_api_connect_timeout_seconds_label": "Таймаут подключения API Remnawave",
+  "admin_settings_field_panel_api_connect_timeout_seconds_description": "Максимальное время получения или открытия соединения с Remnawave API, в секундах.",
+  "admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "TCP/TLS-таймаут API Remnawave",
+  "admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Максимальное время TCP/TLS-подключения к Remnawave API, в секундах.",
+  "admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Таймаут чтения API Remnawave",
+  "admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Максимальное ожидание данных ответа от Remnawave API, в секундах.",
   "admin_settings_field_panel_webhook_secret_label": "Секрет вебхуков Remnawave",
   "admin_settings_field_panel_webhook_secret_description": "Задайте секрет в Remnawave Panel и вставьте то же значение здесь для проверки входящих вебхуков панели.",
   "admin_settings_field_user_squad_uuids_label": "Internal Squads по умолчанию",

tests/test_admin_settings_manifest_i18n.py

@@ -236,10 +236,15 @@ def test_remnawave_settings_include_panel_webhook_metadata():
     remnawave_keys = (
         "PANEL_API_URL",
         "PANEL_API_KEY",
+        "PANEL_API_TOTAL_TIMEOUT_SECONDS",
+        "PANEL_API_CONNECT_TIMEOUT_SECONDS",
+        "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
+        "PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
         "PANEL_WEBHOOK_SECRET",
         "USER_SQUAD_UUIDS",
         "USER_EXTERNAL_SQUAD_UUID",
     )
+    timeout_keys = remnawave_keys[2:6]
 
     assert field["webhook_path"] == "/webhook/panel"
     assert field["webhook_requires_base_url"] is True
@@ -250,10 +255,18 @@ def test_remnawave_settings_include_panel_webhook_metadata():
         assert manifest[setting_key]["section_order"] == 3
         assert manifest[setting_key]["subsection"] is None
 
+    for setting_key in timeout_keys:
+        assert manifest[setting_key]["type"] == "float"
+        assert manifest[setting_key]["optional"] is False
+        assert manifest[setting_key]["min"] == 1
+
     for language in ("ru", "en"):
         messages = _locale(language)
         assert "admin_settings_section_remnawave" in messages
         assert field["webhook_hint_i18n_key"] in messages
+        for setting_key in timeout_keys:
+            assert manifest[setting_key]["i18n_label_key"] in messages
+            assert manifest[setting_key]["i18n_description_key"] in messages
 
 
 def test_payment_provider_admin_only_toggles_are_mutually_exclusive():

tests/test_panel_api_service_logging.py

@@ -1,8 +1,11 @@
 import asyncio
+import time
 import unittest
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, patch
 
+import aiohttp
+
 from bot.services.panel_api_service import PanelApiService
 
 
@@ -16,6 +19,68 @@ class PanelApiServiceLoggingTests(unittest.IsolatedAsyncioTestCase):
             )
         )
 
+    async def test_client_timeout_uses_panel_settings(self):
+        service = PanelApiService(
+            SimpleNamespace(
+                PANEL_API_URL="https://panel.example.test/api",
+                PANEL_API_KEY="panel-key",
+                PANEL_API_TOTAL_TIMEOUT_SECONDS="30",
+                PANEL_API_CONNECT_TIMEOUT_SECONDS="10",
+                PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS="9",
+                PANEL_API_SOCK_READ_TIMEOUT_SECONDS="20",
+            )
+        )
+
+        timeout = service._client_timeout()
+
+        self.assertEqual(timeout.total, 30)
+        self.assertEqual(timeout.connect, 10)
+        self.assertEqual(timeout.sock_connect, 9)
+        self.assertEqual(timeout.sock_read, 20)
+
+    async def test_get_request_retries_connection_timeout(self):
+        service = self._make_service()
+        request_calls = 0
+
+        class OkResponse:
+            status = 200
+            headers = {"Content-Type": "application/json"}
+
+            async def __aenter__(self):
+                return self
+
+            async def __aexit__(self, exc_type, exc_val, exc_tb):
+                return None
+
+            async def text(self):
+                return '{"response": {"ok": true}}'
+
+        def fake_request(*_args, **_kwargs):
+            nonlocal request_calls
+            request_calls += 1
+            if request_calls == 1:
+                raise aiohttp.ConnectionTimeoutError("connect took too long")
+            return OkResponse()
+
+        service._get_session = AsyncMock(return_value=SimpleNamespace(request=fake_request))
+
+        with patch("bot.services.panel_api_service.asyncio.sleep", new=AsyncMock()):
+            result = await service._request("GET", "/internal-squads")
+
+        self.assertEqual(result, {"response": {"ok": True}})
+        self.assertEqual(request_calls, 2)
+
+    async def test_get_internal_squads_uses_stale_cache_after_refresh_failure(self):
+        service = self._make_service()
+        stale_squads = [{"uuid": "squad-1", "name": "Squad 1"}]
+        service._squads_cache._data["list"] = (time.monotonic() - 1, stale_squads)
+        service._get_internal_squads_uncached = AsyncMock(return_value=None)
+
+        squads = await service.get_internal_squads()
+
+        self.assertEqual(squads, stale_squads)
+        service._get_internal_squads_uncached.assert_awaited_once()
+
     async def test_update_user_details_does_not_log_full_response_by_default(self):
         service = self._make_service()
         service._request = AsyncMock(return_value={"response": {"uuid": "user-uuid"}})

tests/test_performance_caches.py

@@ -42,6 +42,14 @@ class AsyncTTLCacheSingleflightTests(unittest.IsolatedAsyncioTestCase):
 
 
 class AsyncTTLCacheInvalidationTests(unittest.IsolatedAsyncioTestCase):
+    def test_get_stale_returns_expired_cacheable_value(self):
+        cache = AsyncTTLCache(ttl_seconds=60)
+        value = {"ok": True}
+        cache._data["same"] = (time.monotonic() - 1, value)
+
+        self.assertIsNone(cache.get_fresh("same"))
+        self.assertEqual(cache.get_stale("same"), value)
+
     async def test_invalidate_remote_deletes_single_redis_key(self):
         settings = SimpleNamespace(REDIS_URL="redis://example", REDIS_KEY_PREFIX="test")
         cache = AsyncTTLCache(ttl_seconds=60, settings=settings, namespace="bench")