austnv/remnawave-minishop
1
0
Fork 0
Code Issues 1 Pull Requests Activity

fix: harden Remnawave panel timeouts

Browse Source
This commit is contained in:
3252a8
2026-06-02 12:29:43 +03:00
parent 56796d9f22
commit 4d7577f4ec
10 changed files with 249 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/bot/app/web/admin_settings_manifest.py
+36
View File
@@ -101,6 +101,42 @@ SETTINGS_MANIFEST: List[SettingField] = [
"Секретный ключ API панели.",
secret=True,
),
SettingField(
"PANEL_API_TOTAL_TIMEOUT_SECONDS",
"float",
"remnawave",
"Panel API total timeout",
"Maximum total time for one Remnawave API request, in seconds.",
optional=False,
min=1,
),
SettingField(
"PANEL_API_CONNECT_TIMEOUT_SECONDS",
"float",
"remnawave",
"Panel API connect timeout",
"Maximum time to get or open a Remnawave API connection, in seconds.",
optional=False,
min=1,
),
SettingField(
"PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
"float",
"remnawave",
"Panel API socket connect timeout",
"Maximum TCP/TLS connection time for Remnawave API, in seconds.",
optional=False,
min=1,
),
SettingField(
"PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
"float",
"remnawave",
"Panel API socket read timeout",
"Maximum time to wait for response data from Remnawave API, in seconds.",
optional=False,
min=1,
),
SettingField(
"PANEL_WEBHOOK_SECRET",
"string",
backend/bot/services/panel_api_service.py
+94 -11
View File
@@ -22,6 +22,11 @@ class PanelApiService:
_TRANSIENT_STATUS_CODES = (-1, -3)
_SAFE_METHODS = frozenset({"GET", "HEAD"})
_RETRY_BACKOFF_SECONDS = 0.5
_MIN_TIMEOUT_SECONDS = 0.1
_DEFAULT_TOTAL_TIMEOUT_SECONDS = 25.0
_DEFAULT_CONNECT_TIMEOUT_SECONDS = 8.0
_DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS = 8.0
_DEFAULT_SOCK_READ_TIMEOUT_SECONDS = 15.0
def __init__(self, settings: Settings):
self.settings = settings
@@ -70,17 +75,46 @@ class PanelApiService:
async def _get_session(self) -> aiohttp.ClientSession:
if self._session is None or self._session.closed:
# Separate connect/read timeouts so a stuck panel does not hold a
# bot worker for the full window; total caps worst-case latency.
timeout = aiohttp.ClientTimeout(
total=15,
connect=3,
sock_connect=3,
sock_read=10,
)
self._session = aiohttp.ClientSession(timeout=timeout)
self._session = aiohttp.ClientSession(timeout=self._client_timeout())
return self._session
@classmethod
def _timeout_setting(cls, settings: Settings, name: str, default: float) -> float:
raw_value = getattr(settings, name, default)
try:
value = float(raw_value)
except (TypeError, ValueError):
return default
if value <= 0:
return default
return max(cls._MIN_TIMEOUT_SECONDS, value)
def _client_timeout(self) -> aiohttp.ClientTimeout:
# Separate connect/read timeouts so a slow panel route has more room,
# while genuinely stuck requests still cannot pin a worker forever.
return aiohttp.ClientTimeout(
total=self._timeout_setting(
self.settings,
"PANEL_API_TOTAL_TIMEOUT_SECONDS",
self._DEFAULT_TOTAL_TIMEOUT_SECONDS,
),
connect=self._timeout_setting(
self.settings,
"PANEL_API_CONNECT_TIMEOUT_SECONDS",
self._DEFAULT_CONNECT_TIMEOUT_SECONDS,
),
sock_connect=self._timeout_setting(
self.settings,
"PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
self._DEFAULT_SOCK_CONNECT_TIMEOUT_SECONDS,
),
sock_read=self._timeout_setting(
self.settings,
"PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
self._DEFAULT_SOCK_READ_TIMEOUT_SECONDS,
),
)
async def close_session(self):
if self._session and not self._session.closed:
await self._session.close()
@@ -121,6 +155,15 @@ class PanelApiService:
for attempt in range(max_attempts):
result = await self._request_once(method, endpoint, log_full_response, **kwargs)
if attempt + 1 < max_attempts and self._is_transient_error(result):
logging.warning(
"Retrying transient Panel API request method=%s endpoint=%s "
"attempt=%s/%s status_code=%s",
method.upper(),
endpoint,
attempt + 1,
max_attempts,
result.get("status_code") if isinstance(result, dict) else None,
)
await asyncio.sleep(self._RETRY_BACKOFF_SECONDS)
continue
return result
@@ -158,8 +201,8 @@ class PanelApiService:
)
except Exception:
log_prefix += f" | Payload: {str(json_payload_for_log)[:300]}..."
started = time.monotonic()
try:
started = time.monotonic()
async with aiohttp_session.request(
method.upper(), url_for_request, headers=headers, **kwargs
) as response:
@@ -228,15 +271,48 @@ class PanelApiService:
return {"error": True, "status_code": response_status, "details": error_details}
except aiohttp.ClientConnectorError as e:
logging.info(
"metric panel_latency_seconds=%.3f method=%s endpoint=%s status=connect_error",
time.monotonic() - started,
method.upper(),
endpoint,
)
logging.error(f"Panel API ClientConnectorError to {url_for_request}: {e}")
return {"error": True, "status_code": -1, "message": f"Connection error: {str(e)}"}
except aiohttp.ServerTimeoutError as e:
logging.info(
"metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout",
time.monotonic() - started,
method.upper(),
endpoint,
)
logging.warning("Panel API timeout to %s: %s", url_for_request, e)
return {"error": True, "status_code": -3, "message": f"Request timed out: {str(e)}"}
except aiohttp.ClientError as e:
logging.info(
"metric panel_latency_seconds=%.3f method=%s endpoint=%s status=client_error",
time.monotonic() - started,
method.upper(),
endpoint,
)
logging.exception("Panel API ClientError to %s.", url_for_request)
return {"error": True, "status_code": -2, "message": f"Client error: {str(e)}"}
except asyncio.TimeoutError:
logging.info(
"metric panel_latency_seconds=%.3f method=%s endpoint=%s status=timeout",
time.monotonic() - started,
method.upper(),
endpoint,
)
logging.error(f"Panel API request to {url_for_request} timed out.")
return {"error": True, "status_code": -3, "message": "Request timed out"}
except Exception as e:
logging.info(
"metric panel_latency_seconds=%.3f method=%s endpoint=%s status=unexpected_error",
time.monotonic() - started,
method.upper(),
endpoint,
)
logging.error(
f"Unexpected Panel API request error to {url_for_request}: {e}", exc_info=True
)
@@ -885,7 +961,14 @@ class PanelApiService:
await self._devices_cache.invalidate_remote(f"user:{user_uuid}")
async def get_internal_squads(self) -> Optional[List[Dict[str, Any]]]:
return await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached)
squads = await self._squads_cache.get_or_load("list", self._get_internal_squads_uncached)
if squads is not None:
return squads
stale_squads = self._squads_cache.get_stale("list")
if stale_squads is not None:
logging.warning("Using stale internal squads cache after panel fetch failed.")
return stale_squads
return None
async def _get_internal_squads_uncached(self) -> Optional[List[Dict[str, Any]]]:
response_data = await self._request("GET", "/internal-squads", log_full_response=False)
backend/bot/utils/ttl_cache.py
+9
View File
@@ -29,6 +29,15 @@ class AsyncTTLCache:
return None
return value
def get_stale(self, key: str) -> Optional[Any]:
entry = self._data.get(key)
if entry is None:
return None
_, value = entry
if not self._is_cacheable(value):
return None
return value
@staticmethod
def _is_cacheable(value: Any) -> bool:
if value is None:
backend/config/settings.py
+4
View File
@@ -100,6 +100,10 @@ class Settings(BaseSettings):
PANEL_DEVICES_CACHE_TTL_SECONDS: int = Field(default=5)
PANEL_ALL_USERS_CACHE_TTL_SECONDS: int = Field(default=5)
PANEL_ALL_USERS_PAGE_SIZE: int = Field(default=1000)
PANEL_API_TOTAL_TIMEOUT_SECONDS: float = Field(default=25)
PANEL_API_CONNECT_TIMEOUT_SECONDS: float = Field(default=8)
PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS: float = Field(default=8)
PANEL_API_SOCK_READ_TIMEOUT_SECONDS: float = Field(default=15)
ADMIN_PANEL_STATS_CACHE_TTL_SECONDS: int = Field(default=15)
ADMIN_DB_STATS_CACHE_TTL_SECONDS: int = Field(default=5)
ADMIN_USERS_LIST_CACHE_TTL_SECONDS: int = Field(default=3)
docs/configuration/env-vars.md
+4
View File
@@ -59,6 +59,10 @@
| `PANEL_DEVICES_CACHE_TTL_SECONDS` | TTL кеша устройств пользователя Remnawave. |
| `PANEL_ALL_USERS_CACHE_TTL_SECONDS` | TTL кеша полных сканов пользователей Remnawave. |
| `PANEL_ALL_USERS_PAGE_SIZE` | Размер страницы Remnawave `/users`. |
| `PANEL_API_TOTAL_TIMEOUT_SECONDS` | Общий timeout запроса к Remnawave API. |
| `PANEL_API_CONNECT_TIMEOUT_SECONDS` | Timeout получения соединения с Remnawave API. |
| `PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS` | Timeout TCP/TLS-подключения к Remnawave API. |
| `PANEL_API_SOCK_READ_TIMEOUT_SECONDS` | Timeout ожидания данных ответа Remnawave API. |
| `ADMIN_PANEL_STATS_CACHE_TTL_SECONDS` | TTL статистики Remnawave в админке. |
| `ADMIN_DB_STATS_CACHE_TTL_SECONDS` | TTL дорогих DB-агрегатов админки. |
| `ADMIN_USERS_LIST_CACHE_TTL_SECONDS` | TTL списка пользователей админки. |
locales/en.json
+8
View File
@@ -1544,6 +1544,14 @@
"admin_settings_field_panel_api_url_description": "For example, https://panel.example.com/api.",
"admin_settings_field_panel_api_key_label": "Remnawave API key",
"admin_settings_field_panel_api_key_description": "Secret API key for the panel.",
"admin_settings_field_panel_api_total_timeout_seconds_label": "Remnawave API total timeout",
"admin_settings_field_panel_api_total_timeout_seconds_description": "Maximum total time for one Remnawave API request, in seconds.",
"admin_settings_field_panel_api_connect_timeout_seconds_label": "Remnawave API connect timeout",
"admin_settings_field_panel_api_connect_timeout_seconds_description": "Maximum time to get or open a Remnawave API connection, in seconds.",
"admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "Remnawave API TCP/TLS timeout",
"admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Maximum TCP/TLS connection time for Remnawave API, in seconds.",
"admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Remnawave API read timeout",
"admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Maximum time to wait for response data from Remnawave API, in seconds.",
"admin_settings_field_panel_webhook_secret_label": "Remnawave webhook secret",
"admin_settings_field_panel_webhook_secret_description": "Set the secret in Remnawave Panel and paste the same value here to verify incoming panel webhooks.",
"admin_settings_field_user_squad_uuids_label": "Default Internal Squads",
locales/ru.json
+8
View File
@@ -1544,6 +1544,14 @@
"admin_settings_field_panel_api_url_description": "Например, https://panel.example.com/api.",
"admin_settings_field_panel_api_key_label": "API-ключ Remnawave",
"admin_settings_field_panel_api_key_description": "Секретный ключ API панели.",
"admin_settings_field_panel_api_total_timeout_seconds_label": "Общий таймаут API Remnawave",
"admin_settings_field_panel_api_total_timeout_seconds_description": "Максимальное время одного запроса к Remnawave API, в секундах.",
"admin_settings_field_panel_api_connect_timeout_seconds_label": "Таймаут подключения API Remnawave",
"admin_settings_field_panel_api_connect_timeout_seconds_description": "Максимальное время получения или открытия соединения с Remnawave API, в секундах.",
"admin_settings_field_panel_api_sock_connect_timeout_seconds_label": "TCP/TLS-таймаут API Remnawave",
"admin_settings_field_panel_api_sock_connect_timeout_seconds_description": "Максимальное время TCP/TLS-подключения к Remnawave API, в секундах.",
"admin_settings_field_panel_api_sock_read_timeout_seconds_label": "Таймаут чтения API Remnawave",
"admin_settings_field_panel_api_sock_read_timeout_seconds_description": "Максимальное ожидание данных ответа от Remnawave API, в секундах.",
"admin_settings_field_panel_webhook_secret_label": "Секрет вебхуков Remnawave",
"admin_settings_field_panel_webhook_secret_description": "Задайте секрет в Remnawave Panel и вставьте то же значение здесь для проверки входящих вебхуков панели.",
"admin_settings_field_user_squad_uuids_label": "Internal Squads по умолчанию",
tests/test_admin_settings_manifest_i18n.py
+13
View File
@@ -236,10 +236,15 @@ def test_remnawave_settings_include_panel_webhook_metadata():
remnawave_keys = (
"PANEL_API_URL",
"PANEL_API_KEY",
"PANEL_API_TOTAL_TIMEOUT_SECONDS",
"PANEL_API_CONNECT_TIMEOUT_SECONDS",
"PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS",
"PANEL_API_SOCK_READ_TIMEOUT_SECONDS",
"PANEL_WEBHOOK_SECRET",
"USER_SQUAD_UUIDS",
"USER_EXTERNAL_SQUAD_UUID",
)
timeout_keys = remnawave_keys[2:6]
assert field["webhook_path"] == "/webhook/panel"
assert field["webhook_requires_base_url"] is True
@@ -250,10 +255,18 @@ def test_remnawave_settings_include_panel_webhook_metadata():
assert manifest[setting_key]["section_order"] == 3
assert manifest[setting_key]["subsection"] is None
for setting_key in timeout_keys:
assert manifest[setting_key]["type"] == "float"
assert manifest[setting_key]["optional"] is False
assert manifest[setting_key]["min"] == 1
for language in ("ru", "en"):
messages = _locale(language)
assert "admin_settings_section_remnawave" in messages
assert field["webhook_hint_i18n_key"] in messages
for setting_key in timeout_keys:
assert manifest[setting_key]["i18n_label_key"] in messages
assert manifest[setting_key]["i18n_description_key"] in messages
def test_payment_provider_admin_only_toggles_are_mutually_exclusive():
tests/test_panel_api_service_logging.py
+65
View File
@@ -1,8 +1,11 @@
import asyncio
import time
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import aiohttp
from bot.services.panel_api_service import PanelApiService
@@ -16,6 +19,68 @@ class PanelApiServiceLoggingTests(unittest.IsolatedAsyncioTestCase):
)
)
async def test_client_timeout_uses_panel_settings(self):
service = PanelApiService(
SimpleNamespace(
PANEL_API_URL="https://panel.example.test/api",
PANEL_API_KEY="panel-key",
PANEL_API_TOTAL_TIMEOUT_SECONDS="30",
PANEL_API_CONNECT_TIMEOUT_SECONDS="10",
PANEL_API_SOCK_CONNECT_TIMEOUT_SECONDS="9",
PANEL_API_SOCK_READ_TIMEOUT_SECONDS="20",
)
)
timeout = service._client_timeout()
self.assertEqual(timeout.total, 30)
self.assertEqual(timeout.connect, 10)
self.assertEqual(timeout.sock_connect, 9)
self.assertEqual(timeout.sock_read, 20)
async def test_get_request_retries_connection_timeout(self):
service = self._make_service()
request_calls = 0
class OkResponse:
status = 200
headers = {"Content-Type": "application/json"}
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
return None
async def text(self):
return '{"response": {"ok": true}}'
def fake_request(*_args, **_kwargs):
nonlocal request_calls
request_calls += 1
if request_calls == 1:
raise aiohttp.ConnectionTimeoutError("connect took too long")
return OkResponse()
service._get_session = AsyncMock(return_value=SimpleNamespace(request=fake_request))
with patch("bot.services.panel_api_service.asyncio.sleep", new=AsyncMock()):
result = await service._request("GET", "/internal-squads")
self.assertEqual(result, {"response": {"ok": True}})
self.assertEqual(request_calls, 2)
async def test_get_internal_squads_uses_stale_cache_after_refresh_failure(self):
service = self._make_service()
stale_squads = [{"uuid": "squad-1", "name": "Squad 1"}]
service._squads_cache._data["list"] = (time.monotonic() - 1, stale_squads)
service._get_internal_squads_uncached = AsyncMock(return_value=None)
squads = await service.get_internal_squads()
self.assertEqual(squads, stale_squads)
service._get_internal_squads_uncached.assert_awaited_once()
async def test_update_user_details_does_not_log_full_response_by_default(self):
service = self._make_service()
service._request = AsyncMock(return_value={"response": {"uuid": "user-uuid"}})
tests/test_performance_caches.py
+8
View File
@@ -42,6 +42,14 @@ class AsyncTTLCacheSingleflightTests(unittest.IsolatedAsyncioTestCase):
class AsyncTTLCacheInvalidationTests(unittest.IsolatedAsyncioTestCase):
def test_get_stale_returns_expired_cacheable_value(self):
cache = AsyncTTLCache(ttl_seconds=60)
value = {"ok": True}
cache._data["same"] = (time.monotonic() - 1, value)
self.assertIsNone(cache.get_fresh("same"))
self.assertEqual(cache.get_stale("same"), value)
async def test_invalidate_remote_deletes_single_redis_key(self):
settings = SimpleNamespace(REDIS_URL="redis://example", REDIS_KEY_PREFIX="test")
cache = AsyncTTLCache(ttl_seconds=60, settings=settings, namespace="bench")