From 4d7577f4ec0398638b457ff294865e3f9df64843 Mon Sep 17 00:00:00 2001 From: 3252a8 <3252a8@proton.me> Date: Tue, 2 Jun 2026 12:29:43 +0300 Subject: [PATCH] fix: harden Remnawave panel timeouts --- .../bot/app/web/admin_settings_manifest.py | 36 ++++++ backend/bot/services/panel_api_service.py | 105 ++++++++++++++++-- backend/bot/utils/ttl_cache.py | 9 ++ backend/config/settings.py | 4 + docs/configuration/env-vars.md | 4 + locales/en.json | 8 ++ locales/ru.json | 8 ++ tests/test_admin_settings_manifest_i18n.py | 13 +++ tests/test_panel_api_service_logging.py | 65 +++++++++++ tests/test_performance_caches.py | 8 ++ 10 files changed, 249 insertions(+), 11 deletions(-) diff --git a/backend/bot/app/web/admin_settings_manifest.py b/backend/bot/app/web/admin_settings_manifest.py index 9c86789..1ebffd0 100644 --- a/backend/bot/app/web/admin_settings_manifest.py +++ b/backend/bot/app/web/admin_settings_manifest.py @@ -101,6 +101,42 @@ SETTINGS_MANIFEST: List[SettingField] = [ "Секретный ключ API панели.", secret=True, ), + SettingField( + "PANEL_API_TOTAL_TIMEOUT_SECONDS", + "float", + "remnawave", + "Panel API total timeout", + "Maximum total time for one Remnawave API request, in seconds.", + optional=False, + min=1, + ), + SettingField( + "PANEL_API_CONNECT_TIMEOUT_SECONDS", + "float", + "remnawave", + "Panel API connect timeout", + "Maximum time to get or open a Remnawave API connection, in seconds.", + optional=False, + min=1, + ), + SettingField( + "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS", + "float", + "remnawave", + "Panel API socket connect timeout", + "Maximum TCP/TLS connection time for Remnawave API, in seconds.", + optional=False, + min=1, + ), + SettingField( + "PANEL_API_SOCK_READ_TIMEOUT_SECONDS", + "float", + "remnawave", + "Panel API socket read timeout", + "Maximum time to wait for response data from Remnawave API, in seconds.", + optional=False, + min=1, + ), SettingField( "PANEL_WEBHOOK_SECRET", "string", diff --git a/backend/bot/services/panel_api_service.py b/backend/bot/services/panel_api_service.py index 4ff5c43..bac4e60 100644 --- a/backend/bot/services/panel_api_service.py +++ b/backend/bot/services/panel_api_service.py @@ -22,6 +22,11 @@ class PanelApiService: _TRANSIENT_STATUS_CODES = (-1, -3) _SAFE_METHODS = frozenset({"GET", "HEAD"}) _RETRY_BACKOFF_SECONDS = 0.5 + _MIN_TIMEOUT_SECONDS = 0.1 + _DEFAULT_TOTAL_TIMEOUT_SECONDS = 25.0 + _DEFAULT_CONNECT_TIMEOUT_SECONDS = 8.0 + _DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS = 8.0 + _DEFAULT_SOCK_READ_TIMEOUT_SECONDS = 15.0 def __init__(self, settings: Settings): self.settings = settings @@ -70,17 +75,46 @@ class PanelApiService: async def _get_session(self) -> aiohttp.ClientSession: if self._session is None or self._session.closed: - # Separate connect/read timeouts so a stuck panel does not hold a - # bot worker for the full window; total caps worst-case latency. - timeout = aiohttp.ClientTimeout( - total=15, - connect=3, - sock_connect=3, - sock_read=10, - ) - self._session = aiohttp.ClientSession(timeout=timeout) + self._session = aiohttp.ClientSession(timeout=self._client_timeout()) return self._session + @classmethod + def _timeout_setting(cls, settings: Settings, name: str, default: float) -> float: + raw_value = getattr(settings, name, default) + try: + value = float(raw_value) + except (TypeError, ValueError): + return default + if value <= 0: + return default + return max(cls._MIN_TIMEOUT_SECONDS, value) + + def _client_timeout(self) -> aiohttp.ClientTimeout: + # Separate connect/read timeouts so a slow panel route has more room, + # while genuinely stuck requests still cannot pin a worker forever. + return aiohttp.ClientTimeout( + total=self._timeout_setting( + self.settings, + "PANEL_API_TOTAL_TIMEOUT_SECONDS", + self._DEFAULT_TOTAL_TIMEOUT_SECONDS, + ), + connect=self._timeout_setting( + self.settings, + "PANEL_API_CONNECT_TIMEOUT_SECONDS", + self._DEFAULT_CONNECT_TIMEOUT_SECONDS, + ), + sock_connect=self._timeout_setting( + self.settings, + "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS", + self._DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS, + ), + sock_read=self._timeout_setting( + self.settings, + "PANEL_API_SOCK_READ_TIMEOUT_SECONDS", + self._DEFAULT_SOCK_READ_TIMEOUT_SECONDS, + ), + ) + async def close_session(self): if self._session and not self._session.closed: await self._session.close() @@ -121,6 +155,15 @@ class PanelApiService: for attempt in range(max_attempts): result = await self._request_once(method, endpoint, log_full_response, **kwargs) if attempt + 1 < max_attempts and self._is_transient_error(result): + logging.warning( + "Retrying transient Panel API request method=%s endpoint=%s " + "attempt=%s/%s status_code=%s", + method.upper(), + endpoint, + attempt + 1, + max_attempts, + result.get("status_code") if isinstance(result, dict) else None, + ) await asyncio.sleep(self._RETRY_BACKOFF_SECONDS) continue return result @@ -158,8 +201,8 @@ class PanelApiService: ) except Exception: log_prefix += f" | Payload: {str(json_payload_for_log)[:300]}..." + started = time.monotonic() try: - started = time.monotonic() async with aiohttp_session.request( method.upper(), url_for_request, headers=headers, **kwargs ) as response: @@ -228,15 +271,48 @@ class PanelApiService: return {"error": True, "status_code": response_status, "details": error_details} except aiohttp.ClientConnectorError as e: + logging.info( + "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=connect_error", + time.monotonic() - started, + method.upper(), + endpoint, + ) logging.error(f"Panel API ClientConnectorError to {url_for_request}: {e}") return {"error": True, "status_code": -1, "message": f"Connection error: {str(e)}"} + except aiohttp.ServerTimeoutError as e: + logging.info( + "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout", + time.monotonic() - started, + method.upper(), + endpoint, + ) + logging.warning("Panel API timeout to %s: %s", url_for_request, e) + return {"error": True, "status_code": -3, "message": f"Request timed out: {str(e)}"} except aiohttp.ClientError as e: + logging.info( + "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=client_error", + time.monotonic() - started, + method.upper(), + endpoint, + ) logging.exception("Panel API ClientError to %s.", url_for_request) return {"error": True, "status_code": -2, "message": f"Client error: {str(e)}"} except asyncio.TimeoutError: + logging.info( + "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout", + time.monotonic() - started, + method.upper(), + endpoint, + ) logging.error(f"Panel API request to {url_for_request} timed out.") return {"error": True, "status_code": -3, "message": "Request timed out"} except Exception as e: + logging.info( + "metric panel_latency_seconds=%.3f method=%s endpoint=%s status=unexpected_error", + time.monotonic() - started, + method.upper(), + endpoint, + ) logging.error( f"Unexpected Panel API request error to {url_for_request}: {e}", exc_info=True ) @@ -885,7 +961,14 @@ class PanelApiService: await self._devices_cache.invalidate_remote(f"user:{user_uuid}") async def get_internal_squads(self) -> Optional[List[Dict[str, Any]]]: - return await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached) + squads = await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached) + if squads is not None: + return squads + stale_squads = self._squads_cache.get_stale("list") + if stale_squads is not None: + logging.warning("Using stale internal squads cache after panel fetch failed.") + return stale_squads + return None async def _get_internal_squads_uncached(self) -> Optional[List[Dict[str, Any]]]: response_data = await self._request("GET", "/internal-squads", log_full_response=False) diff --git a/backend/bot/utils/ttl_cache.py b/backend/bot/utils/ttl_cache.py index 18408fd..d182ae7 100644 --- a/backend/bot/utils/ttl_cache.py +++ b/backend/bot/utils/ttl_cache.py @@ -29,6 +29,15 @@ class AsyncTTLCache: return None return value + def get_stale(self, key: str) -> Optional[Any]: + entry = self._data.get(key) + if entry is None: + return None + _, value = entry + if not self._is_cacheable(value): + return None + return value + @staticmethod def _is_cacheable(value: Any) -> bool: if value is None: diff --git a/backend/config/settings.py b/backend/config/settings.py index bd09406..72827f1 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -100,6 +100,10 @@ class Settings(BaseSettings): PANEL_DEVICES_CACHE_TTL_SECONDS: int = Field(default=5) PANEL_ALL_USERS_CACHE_TTL_SECONDS: int = Field(default=5) PANEL_ALL_USERS_PAGE_SIZE: int = Field(default=1000) + PANEL_API_TOTAL_TIMEOUT_SECONDS: float = Field(default=25) + PANEL_API_CONNECT_TIMEOUT_SECONDS: float = Field(default=8) + PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS: float = Field(default=8) + PANEL_API_SOCK_READ_TIMEOUT_SECONDS: float = Field(default=15) ADMIN_PANEL_STATS_CACHE_TTL_SECONDS: int = Field(default=15) ADMIN_DB_STATS_CACHE_TTL_SECONDS: int = Field(default=5) ADMIN_USERS_LIST_CACHE_TTL_SECONDS: int = Field(default=3) diff --git a/docs/configuration/env-vars.md b/docs/configuration/env-vars.md index b7dbdf7..c3f988c 100644 --- a/docs/configuration/env-vars.md +++ b/docs/configuration/env-vars.md @@ -59,6 +59,10 @@ | `PANEL_DEVICES_CACHE_TTL_SECONDS` | TTL кеша устройств пользователя Remnawave. | | `PANEL_ALL_USERS_CACHE_TTL_SECONDS` | TTL кеша полных сканов пользователей Remnawave. | | `PANEL_ALL_USERS_PAGE_SIZE` | Размер страницы Remnawave `/users`. | +| `PANEL_API_TOTAL_TIMEOUT_SECONDS` | Общий timeout запроса к Remnawave API. | +| `PANEL_API_CONNECT_TIMEOUT_SECONDS` | Timeout получения соединения с Remnawave API. | +| `PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS` | Timeout TCP/TLS-подключения к Remnawave API. | +| `PANEL_API_SOCK_READ_TIMEOUT_SECONDS` | Timeout ожидания данных ответа Remnawave API. | | `ADMIN_PANEL_STATS_CACHE_TTL_SECONDS` | TTL статистики Remnawave в админке. | | `ADMIN_DB_STATS_CACHE_TTL_SECONDS` | TTL дорогих DB-агрегатов админки. | | `ADMIN_USERS_LIST_CACHE_TTL_SECONDS` | TTL списка пользователей админки. | diff --git a/locales/en.json b/locales/en.json index cf3ffff..360a129 100644 --- a/locales/en.json +++ b/locales/en.json @@ -1544,6 +1544,14 @@ "admin_settings_field_panel_api_url_description": "For example, https://panel.example.com/api.", "admin_settings_field_panel_api_key_label": "Remnawave API key", "admin_settings_field_panel_api_key_description": "Secret API key for the panel.", + "admin_settings_field_panel_api_total_timeout_seconds_label": "Remnawave API total timeout", + "admin_settings_field_panel_api_total_timeout_seconds_description": "Maximum total time for one Remnawave API request, in seconds.", + "admin_settings_field_panel_api_connect_timeout_seconds_label": "Remnawave API connect timeout", + "admin_settings_field_panel_api_connect_timeout_seconds_description": "Maximum time to get or open a Remnawave API connection, in seconds.", + "admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "Remnawave API TCP/TLS timeout", + "admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Maximum TCP/TLS connection time for Remnawave API, in seconds.", + "admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Remnawave API read timeout", + "admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Maximum time to wait for response data from Remnawave API, in seconds.", "admin_settings_field_panel_webhook_secret_label": "Remnawave webhook secret", "admin_settings_field_panel_webhook_secret_description": "Set the secret in Remnawave Panel and paste the same value here to verify incoming panel webhooks.", "admin_settings_field_user_squad_uuids_label": "Default Internal Squads", diff --git a/locales/ru.json b/locales/ru.json index 4ce0660..ab47d7f 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -1544,6 +1544,14 @@ "admin_settings_field_panel_api_url_description": "Например, https://panel.example.com/api.", "admin_settings_field_panel_api_key_label": "API-ключ Remnawave", "admin_settings_field_panel_api_key_description": "Секретный ключ API панели.", + "admin_settings_field_panel_api_total_timeout_seconds_label": "Общий таймаут API Remnawave", + "admin_settings_field_panel_api_total_timeout_seconds_description": "Максимальное время одного запроса к Remnawave API, в секундах.", + "admin_settings_field_panel_api_connect_timeout_seconds_label": "Таймаут подключения API Remnawave", + "admin_settings_field_panel_api_connect_timeout_seconds_description": "Максимальное время получения или открытия соединения с Remnawave API, в секундах.", + "admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "TCP/TLS-таймаут API Remnawave", + "admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Максимальное время TCP/TLS-подключения к Remnawave API, в секундах.", + "admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Таймаут чтения API Remnawave", + "admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Максимальное ожидание данных ответа от Remnawave API, в секундах.", "admin_settings_field_panel_webhook_secret_label": "Секрет вебхуков Remnawave", "admin_settings_field_panel_webhook_secret_description": "Задайте секрет в Remnawave Panel и вставьте то же значение здесь для проверки входящих вебхуков панели.", "admin_settings_field_user_squad_uuids_label": "Internal Squads по умолчанию", diff --git a/tests/test_admin_settings_manifest_i18n.py b/tests/test_admin_settings_manifest_i18n.py index 2144100..fb11359 100644 --- a/tests/test_admin_settings_manifest_i18n.py +++ b/tests/test_admin_settings_manifest_i18n.py @@ -236,10 +236,15 @@ def test_remnawave_settings_include_panel_webhook_metadata(): remnawave_keys = ( "PANEL_API_URL", "PANEL_API_KEY", + "PANEL_API_TOTAL_TIMEOUT_SECONDS", + "PANEL_API_CONNECT_TIMEOUT_SECONDS", + "PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS", + "PANEL_API_SOCK_READ_TIMEOUT_SECONDS", "PANEL_WEBHOOK_SECRET", "USER_SQUAD_UUIDS", "USER_EXTERNAL_SQUAD_UUID", ) + timeout_keys = remnawave_keys[2:6] assert field["webhook_path"] == "/webhook/panel" assert field["webhook_requires_base_url"] is True @@ -250,10 +255,18 @@ def test_remnawave_settings_include_panel_webhook_metadata(): assert manifest[setting_key]["section_order"] == 3 assert manifest[setting_key]["subsection"] is None + for setting_key in timeout_keys: + assert manifest[setting_key]["type"] == "float" + assert manifest[setting_key]["optional"] is False + assert manifest[setting_key]["min"] == 1 + for language in ("ru", "en"): messages = _locale(language) assert "admin_settings_section_remnawave" in messages assert field["webhook_hint_i18n_key"] in messages + for setting_key in timeout_keys: + assert manifest[setting_key]["i18n_label_key"] in messages + assert manifest[setting_key]["i18n_description_key"] in messages def test_payment_provider_admin_only_toggles_are_mutually_exclusive(): diff --git a/tests/test_panel_api_service_logging.py b/tests/test_panel_api_service_logging.py index 4691bb5..8589496 100644 --- a/tests/test_panel_api_service_logging.py +++ b/tests/test_panel_api_service_logging.py @@ -1,8 +1,11 @@ import asyncio +import time import unittest from types import SimpleNamespace from unittest.mock import AsyncMock, patch +import aiohttp + from bot.services.panel_api_service import PanelApiService @@ -16,6 +19,68 @@ class PanelApiServiceLoggingTests(unittest.IsolatedAsyncioTestCase): ) ) + async def test_client_timeout_uses_panel_settings(self): + service = PanelApiService( + SimpleNamespace( + PANEL_API_URL="https://panel.example.test/api", + PANEL_API_KEY="panel-key", + PANEL_API_TOTAL_TIMEOUT_SECONDS="30", + PANEL_API_CONNECT_TIMEOUT_SECONDS="10", + PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS="9", + PANEL_API_SOCK_READ_TIMEOUT_SECONDS="20", + ) + ) + + timeout = service._client_timeout() + + self.assertEqual(timeout.total, 30) + self.assertEqual(timeout.connect, 10) + self.assertEqual(timeout.sock_connect, 9) + self.assertEqual(timeout.sock_read, 20) + + async def test_get_request_retries_connection_timeout(self): + service = self._make_service() + request_calls = 0 + + class OkResponse: + status = 200 + headers = {"Content-Type": "application/json"} + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + return None + + async def text(self): + return '{"response": {"ok": true}}' + + def fake_request(*_args, **_kwargs): + nonlocal request_calls + request_calls += 1 + if request_calls == 1: + raise aiohttp.ConnectionTimeoutError("connect took too long") + return OkResponse() + + service._get_session = AsyncMock(return_value=SimpleNamespace(request=fake_request)) + + with patch("bot.services.panel_api_service.asyncio.sleep", new=AsyncMock()): + result = await service._request("GET", "/internal-squads") + + self.assertEqual(result, {"response": {"ok": True}}) + self.assertEqual(request_calls, 2) + + async def test_get_internal_squads_uses_stale_cache_after_refresh_failure(self): + service = self._make_service() + stale_squads = [{"uuid": "squad-1", "name": "Squad 1"}] + service._squads_cache._data["list"] = (time.monotonic() - 1, stale_squads) + service._get_internal_squads_uncached = AsyncMock(return_value=None) + + squads = await service.get_internal_squads() + + self.assertEqual(squads, stale_squads) + service._get_internal_squads_uncached.assert_awaited_once() + async def test_update_user_details_does_not_log_full_response_by_default(self): service = self._make_service() service._request = AsyncMock(return_value={"response": {"uuid": "user-uuid"}}) diff --git a/tests/test_performance_caches.py b/tests/test_performance_caches.py index 6093678..1d23a03 100644 --- a/tests/test_performance_caches.py +++ b/tests/test_performance_caches.py @@ -42,6 +42,14 @@ class AsyncTTLCacheSingleflightTests(unittest.IsolatedAsyncioTestCase): class AsyncTTLCacheInvalidationTests(unittest.IsolatedAsyncioTestCase): + def test_get_stale_returns_expired_cacheable_value(self): + cache = AsyncTTLCache(ttl_seconds=60) + value = {"ok": True} + cache._data["same"] = (time.monotonic() - 1, value) + + self.assertIsNone(cache.get_fresh("same")) + self.assertEqual(cache.get_stale("same"), value) + async def test_invalidate_remote_deletes_single_redis_key(self): settings = SimpleNamespace(REDIS_URL="redis://example", REDIS_KEY_PREFIX="test") cache = AsyncTTLCache(ttl_seconds=60, settings=settings, namespace="bench")