austnv/remnawave-minishop
1
0
Fork 0
Code Issues 1 Pull Requests Activity

refactor: improve startup and sync performance

Browse Source
This commit is contained in:
3252a8
2026-05-20 23:04:29 +03:00
parent 8192eaf55b
commit a7f298743d
12 changed files with 973 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/perf_benchmarks.py
+263 -6
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import argparse
import asyncio
import json
import math
import sys
import time
from pathlib import Path
@@ -16,7 +15,15 @@ for path in (str(BACKEND), str(ROOT)):
if path not in sys.path:
sys.path.insert(0, path)
import bot.app.web.subscription_webapp # noqa: E402,F401
from bot.app.web.admin_api_impl import stats as admin_stats_module # noqa: E402
from bot.handlers.admin.sync_admin import ( # noqa: E402
_description_matches,
_subscription_update_delta,
)
from bot.middlewares import profile_sync as profile_sync_module # noqa: E402
from bot.services import panel_api_service # noqa: E402
from bot.services.panel_api_service import PanelApiService # noqa: E402
from bot.services.tariff_worker import TariffTrafficWorker # noqa: E402
from bot.utils import config_link # noqa: E402
from bot.utils.config_link import prepare_config_links # noqa: E402
@@ -25,6 +32,14 @@ from bot.utils.ttl_cache import AsyncTTLCache # noqa: E402
DEFAULT_USER_SIZES = (200, 500, 1000, 5000, 10000)
def estimated_panel_user_pages(users: int, page_size: int = 100) -> int:
if users <= 0:
return 1
# get_all_panel_users stops on a short/empty page, so exact page multiples
# need one final empty-page request.
return users // page_size + 1
class FakePanel:
def __init__(self, users: int):
self.calls = 0
@@ -84,7 +99,7 @@ async def bench_premium_usage(users: int) -> dict:
async def bench_panel_user_prefetch(users: int) -> dict:
panel = FakeBulkPanel(users)
worker = TariffTrafficWorker(
settings=SimpleNamespace(TARIFF_WORKER_BULK_PANEL_FETCH_THRESHOLD=200),
settings=SimpleNamespace(TARIFF_WORKER_BULK_PANEL_FETCH_THRESHOLD=50),
session_factory=SimpleNamespace(),
panel_service=panel,
subscription_service=SimpleNamespace(),
@@ -98,7 +113,156 @@ async def bench_panel_user_prefetch(users: int) -> dict:
"service_calls": panel.calls,
"matched": len(by_uuid or {}),
"legacy_user_get_calls": users,
"estimated_bulk_http_pages_at_100": math.ceil(users / 100),
"estimated_bulk_http_pages_at_100": estimated_panel_user_pages(users),
}
async def bench_panel_sync_startup(users: int) -> dict:
end_date = "2026-06-20T12:00:00+00:00"
from datetime import datetime, timezone
parsed_end_date = datetime.fromisoformat(end_date).astimezone(timezone.utc)
started = time.perf_counter()
subscription_writes = 0
description_patches = 0
for index in range(users):
desired_description = f"user{index}@example.test\nusername_{index}"
current_description = f"user{index}@example.test username_{index}"
if not _description_matches(current_description, desired_description):
description_patches += 1
subscription = SimpleNamespace(
user_id=index,
panel_user_uuid=f"panel-{index}",
end_date=parsed_end_date,
is_active=True,
status_from_panel="ACTIVE",
)
delta = _subscription_update_delta(
subscription,
{
"user_id": index,
"panel_user_uuid": f"panel-{index}",
"end_date": parsed_end_date,
"is_active": True,
"status_from_panel": "ACTIVE",
},
)
if delta:
subscription_writes += 1
elapsed = time.perf_counter() - started
return {
"seconds": elapsed,
"panel_get_pages_estimate": estimated_panel_user_pages(users),
"legacy_user_lookup_queries_estimate": users * 3,
"optimized_user_lookup_queries_estimate": 1,
"legacy_subscription_lookup_queries_estimate": users,
"optimized_subscription_lookup_queries_estimate": 1,
"legacy_subscription_write_attempts": users,
"optimized_subscription_writes": subscription_writes,
"description_panel_patches": description_patches,
}
async def bench_panel_user_cache(users: int) -> dict:
settings = SimpleNamespace(
PANEL_API_URL="https://panel.example.test/api",
PANEL_API_KEY="key",
USER_HWID_DEVICE_LIMIT=None,
PANEL_USER_CACHE_TTL_SECONDS=60,
PANEL_DEVICES_CACHE_TTL_SECONDS=60,
PANEL_ALL_USERS_CACHE_TTL_SECONDS=60,
REDIS_URL=None,
REDIS_KEY_PREFIX="bench",
)
service = PanelApiService(settings)
calls = 0
async def fake_request(method, endpoint, log_full_response=False, **kwargs):
nonlocal calls
calls += 1
await asyncio.sleep(0.001)
return {"response": {"uuid": "panel-user", "username": "cached"}}
service._request = fake_request
started = time.perf_counter()
await asyncio.gather(*(service.get_user_by_uuid("panel-user") for _ in range(users)))
elapsed = time.perf_counter() - started
return {
"seconds": elapsed,
"panel_calls": calls,
"legacy_panel_calls": users,
}
async def bench_panel_all_users_cache(users: int) -> dict:
settings = SimpleNamespace(
PANEL_API_URL="https://panel.example.test/api",
PANEL_API_KEY="key",
USER_HWID_DEVICE_LIMIT=None,
PANEL_USER_CACHE_TTL_SECONDS=60,
PANEL_DEVICES_CACHE_TTL_SECONDS=60,
PANEL_ALL_USERS_CACHE_TTL_SECONDS=60,
REDIS_URL=None,
REDIS_KEY_PREFIX="bench",
)
service = PanelApiService(settings)
panel_users = [{"uuid": f"panel-{index}"} for index in range(users)]
calls = 0
async def fake_request(method, endpoint, log_full_response=False, **kwargs):
nonlocal calls
calls += 1
await asyncio.sleep(0.001)
params = kwargs.get("params") or {}
size = int(params.get("size", 100))
start = int(params.get("start", 0))
return {"response": {"users": panel_users[start : start + size]}}
service._request = fake_request
started = time.perf_counter()
first, second = await asyncio.gather(
service.get_all_panel_users(),
service.get_all_panel_users(),
)
elapsed = time.perf_counter() - started
pages = estimated_panel_user_pages(users)
return {
"seconds": elapsed,
"panel_calls": calls,
"legacy_panel_calls": pages * 2,
"users_first": len(first or []),
"users_second": len(second or []),
}
async def bench_panel_devices_cache(users: int) -> dict:
settings = SimpleNamespace(
PANEL_API_URL="https://panel.example.test/api",
PANEL_API_KEY="key",
USER_HWID_DEVICE_LIMIT=None,
PANEL_USER_CACHE_TTL_SECONDS=60,
PANEL_DEVICES_CACHE_TTL_SECONDS=60,
PANEL_ALL_USERS_CACHE_TTL_SECONDS=60,
REDIS_URL=None,
REDIS_KEY_PREFIX="bench",
)
service = PanelApiService(settings)
calls = 0
async def fake_request(method, endpoint, log_full_response=False, **kwargs):
nonlocal calls
calls += 1
await asyncio.sleep(0.001)
return {"response": [{"hwid": "device-1"}]}
service._request = fake_request
started = time.perf_counter()
await asyncio.gather(*(service.get_user_devices("panel-user") for _ in range(users)))
elapsed = time.perf_counter() - started
return {
"seconds": elapsed,
"panel_calls": calls,
"legacy_panel_calls": users,
}
@@ -132,6 +296,83 @@ async def bench_ttl_singleflight(users: int) -> dict:
}
class FakeAdminStatsPanel:
def __init__(self):
self.calls = {
"system": 0,
"bandwidth": 0,
"nodes": 0,
"nodes_bandwidth": 0,
"online": 0,
}
async def get_system_stats(self):
self.calls["system"] += 1
await asyncio.sleep(0.001)
return {"users": {"totalUsers": 10}}
async def get_bandwidth_stats(self):
self.calls["bandwidth"] += 1
await asyncio.sleep(0.001)
return {"current": 123}
async def get_nodes_statistics(self):
self.calls["nodes"] += 1
await asyncio.sleep(0.001)
return {"nodes": []}
async def get_nodes_bandwidth_usage(self, *, start: str, end: str, top_nodes_limit: int = 64):
self.calls["nodes_bandwidth"] += 1
await asyncio.sleep(0.001)
return {"topNodes": []}
async def get_nodes_online_lookups(self):
self.calls["online"] += 1
await asyncio.sleep(0.001)
return {"byUuid": {}, "byName": {}}
async def bench_admin_stats_cache(users: int) -> dict:
admin_stats_module._ADMIN_PANEL_STATS_CACHES.clear()
settings = SimpleNamespace(
ADMIN_PANEL_STATS_CACHE_TTL_SECONDS=60,
REDIS_URL=None,
REDIS_KEY_PREFIX="bench",
)
panel = FakeAdminStatsPanel()
started = time.perf_counter()
await asyncio.gather(
*(admin_stats_module._load_admin_panel_stats(None, settings, panel) for _ in range(users))
)
elapsed = time.perf_counter() - started
return {
"seconds": elapsed,
"panel_endpoint_calls": sum(panel.calls.values()),
"legacy_panel_endpoint_calls": users * len(panel.calls),
}
async def bench_profile_sync_guard(users: int) -> dict:
profile_sync_module._LOCAL_PROFILE_SYNC_CHECKS.clear()
settings = SimpleNamespace(
PROFILE_SYNC_CACHE_TTL_SECONDS=900,
REDIS_URL=None,
REDIS_KEY_PREFIX="bench",
)
allowed_checks = 0
started = time.perf_counter()
for _ in range(users):
if not await profile_sync_module._profile_sync_recently_checked(settings, 42):
allowed_checks += 1
await profile_sync_module._mark_profile_sync_checked(settings, 42)
elapsed = time.perf_counter() - started
return {
"seconds": elapsed,
"profile_checks_allowed": allowed_checks,
"legacy_profile_checks": users,
}
async def bench_crypt4(users: int) -> dict:
config_link._CRYPT4_LINK_CACHES.clear()
settings = SimpleNamespace(
@@ -175,9 +416,15 @@ async def run_suite(user_sizes: tuple[int, ...]) -> dict:
results: dict[str, dict] = {}
for users in user_sizes:
results[str(users)] = {
"panel_sync_startup": await bench_panel_sync_startup(users),
"panel_user_bulk_prefetch": await bench_panel_user_prefetch(users),
"panel_all_users_cache": await bench_panel_all_users_cache(users),
"panel_user_cache": await bench_panel_user_cache(users),
"panel_devices_cache": await bench_panel_devices_cache(users),
"premium_usage_1_node": await bench_premium_usage(users),
"ttl_cache_cold_single_key": await bench_ttl_singleflight(users),
"admin_stats_cache": await bench_admin_stats_cache(users),
"profile_sync_guard": await bench_profile_sync_guard(users),
"crypt4_same_link": await bench_crypt4(users),
}
return results
@@ -186,16 +433,26 @@ async def run_suite(user_sizes: tuple[int, ...]) -> dict:
def _print_table(results: dict[str, dict]) -> None:
print(
"users | bulk_pages_est | premium_usage_s | premium_panel_calls | "
"ttl_loader_calls | crypt4_panel_calls"
"sync_db_reads_est | sync_db_writes | user_cache_calls | "
"all_users_calls | device_cache_calls | admin_panel_calls | crypt4_panel_calls"
)
print("-" * 104)
print("-" * 154)
for users, data in results.items():
sync_optimized_reads = (
data["panel_sync_startup"]["optimized_user_lookup_queries_estimate"]
+ data["panel_sync_startup"]["optimized_subscription_lookup_queries_estimate"]
)
print(
f"{users:>5} | "
f"{data['panel_user_bulk_prefetch']['estimated_bulk_http_pages_at_100']:>14} | "
f"{data['premium_usage_1_node']['seconds']:>15.6f} | "
f"{data['premium_usage_1_node']['panel_calls']:>19} | "
f"{data['ttl_cache_cold_single_key']['loader_calls']:>16} | "
f"{sync_optimized_reads:>17} | "
f"{data['panel_sync_startup']['optimized_subscription_writes']:>14} | "
f"{data['panel_user_cache']['panel_calls']:>16} | "
f"{data['panel_all_users_cache']['panel_calls']:>15} | "
f"{data['panel_devices_cache']['panel_calls']:>18} | "
f"{data['admin_stats_cache']['panel_endpoint_calls']:>17} | "
f"{data['crypt4_same_link']['panel_calls']:>18}"
)