austnv/remnawave-minishop
1
0
Fork 0
Code Issues 1 Pull Requests Activity

refactor: improve panel sync performance

Browse Source
This commit is contained in:
3252a8
2026-05-20 23:18:03 +03:00
parent a7f298743d
commit 5001185bf8
11 changed files with 230 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/bot/handlers/admin/sync_admin.py
+78 -18
View File
@@ -55,6 +55,56 @@ def _datetime_matches(current: Optional[datetime], desired: datetime) -> bool:
return abs(delta.total_seconds()) < 1
def _as_utc(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def _should_update_lifetime_used_traffic(
existing_user,
lifetime_used: int,
*,
now: datetime,
settings: Settings,
is_duplicate_panel_identity: bool = False,
) -> bool:
if is_duplicate_panel_identity:
return False
current_value = existing_user.lifetime_used_traffic_bytes
if current_value == lifetime_used:
return False
if current_value is None:
return True
try:
min_delta_bytes = max(
0,
int(getattr(settings, "PANEL_SYNC_LIFETIME_TRAFFIC_MIN_DELTA_BYTES", 0) or 0),
)
except (TypeError, ValueError):
min_delta_bytes = 0
if min_delta_bytes and abs(int(lifetime_used) - int(current_value or 0)) >= min_delta_bytes:
return True
try:
min_interval_seconds = max(
0,
int(getattr(settings, "PANEL_SYNC_LIFETIME_TRAFFIC_MIN_INTERVAL_SECONDS", 0) or 0),
)
except (TypeError, ValueError):
min_interval_seconds = 0
if min_interval_seconds <= 0:
return True
last_synced_at = getattr(existing_user, "lifetime_used_traffic_synced_at", None)
if not last_synced_at:
return True
return (_as_utc(now) - _as_utc(last_synced_at)).total_seconds() >= min_interval_seconds
def _subscription_update_delta(
subscription: Subscription, desired: dict[str, Any]
) -> dict[str, Any]:
@@ -469,6 +519,7 @@ async def _perform_sync_impl(
# Get the actual user_id for subscription operations
actual_user_id = existing_user.user_id
is_duplicate_panel_identity = False
# Update panel UUID if different
if existing_user.panel_user_uuid != panel_uuid:
@@ -480,6 +531,7 @@ async def _perform_sync_impl(
in panel_uuids_by_telegram_id.get(telegram_id_from_panel, set())
)
if linked_uuid_still_present:
is_duplicate_panel_identity = True
logging.warning(
"Sync: duplicate panel users share telegramId %s; keeping local panel UUID %s and skipping duplicate panel UUID %s.", # noqa: E501
telegram_id_from_panel,
@@ -492,32 +544,40 @@ async def _perform_sync_impl(
users_uuid_updated += 1
users_by_panel_uuid[panel_uuid] = existing_user
logging.info(f"Updated panel UUID for user {actual_user_id}: {panel_uuid}")
existing_user, email_was_bound = await _bind_panel_email_to_user(
session,
existing_user=existing_user,
email_from_panel=email_from_panel,
panel_uuid=panel_uuid,
)
if email_was_bound:
user_was_updated = True
if email_from_panel:
users_by_email[email_from_panel] = existing_user
if telegram_id_from_panel and existing_user.telegram_id != telegram_id_from_panel:
existing_user.telegram_id = telegram_id_from_panel
user_was_updated = True
users_by_telegram_id[telegram_id_from_panel] = existing_user
if not is_duplicate_panel_identity:
existing_user, email_was_bound = await _bind_panel_email_to_user(
session,
existing_user=existing_user,
email_from_panel=email_from_panel,
panel_uuid=panel_uuid,
)
if email_was_bound:
user_was_updated = True
if email_from_panel:
users_by_email[email_from_panel] = existing_user
if (
telegram_id_from_panel
and existing_user.telegram_id != telegram_id_from_panel
):
existing_user.telegram_id = telegram_id_from_panel
user_was_updated = True
users_by_telegram_id[telegram_id_from_panel] = existing_user
lifetime_used = _extract_lifetime_used_traffic_bytes(panel_user_dict)
if (
lifetime_used is not None
and existing_user.lifetime_used_traffic_bytes != lifetime_used
if lifetime_used is not None and _should_update_lifetime_used_traffic(
existing_user,
lifetime_used,
now=datetime.now(timezone.utc),
settings=settings,
is_duplicate_panel_identity=is_duplicate_panel_identity,
):
existing_user.lifetime_used_traffic_bytes = lifetime_used
existing_user.lifetime_used_traffic_synced_at = datetime.now(timezone.utc)
user_was_updated = True
# Ensure panel description contains Telegram fields
try:
if panel_uuid and existing_user:
if panel_uuid and existing_user and not is_duplicate_panel_identity:
description_text = "\n".join(
line
for line in [
backend/bot/services/panel_api_service.py
+40 -6
View File
@@ -242,20 +242,54 @@ class PanelApiService:
)
return {"error": True, "status_code": -4, "message": f"Unexpected error: {str(e)}"}
def _resolve_all_users_page_size(self, page_size: Optional[int] = None) -> int:
raw_value = (
page_size
if page_size is not None
else getattr(self.settings, "PANEL_ALL_USERS_PAGE_SIZE", 1000)
)
try:
value = int(raw_value or 1000)
except (TypeError, ValueError):
value = 1000
return min(1000, max(1, value))
async def get_all_panel_users(
self, page_size: int = 100, log_responses: bool = False
self, page_size: Optional[int] = None, log_responses: bool = False
) -> Optional[List[Dict[str, Any]]]:
if log_responses or page_size != 100 or self._all_users_cache.ttl_seconds <= 0:
resolved_page_size = self._resolve_all_users_page_size(page_size)
if log_responses or self._all_users_cache.ttl_seconds <= 0:
return await self._get_all_panel_users_uncached(
page_size=page_size, log_responses=log_responses
page_size=resolved_page_size, log_responses=log_responses
)
return await self._all_users_cache.get_or_load(
f"page_size:{page_size}",
lambda: self._get_all_panel_users_uncached(page_size=page_size, log_responses=False),
f"page_size:{resolved_page_size}",
lambda: self._get_all_panel_users_uncached(
page_size=resolved_page_size, log_responses=False
),
)
async def _get_all_panel_users_uncached(
self, page_size: int = 100, log_responses: bool = False
self, page_size: Optional[int] = None, log_responses: bool = False
) -> Optional[List[Dict[str, Any]]]:
resolved_page_size = self._resolve_all_users_page_size(page_size)
users = await self._fetch_all_panel_users_pages(
page_size=resolved_page_size,
log_responses=log_responses,
)
if users is None and resolved_page_size != 100:
logging.warning(
"Panel API users fetch failed with page size %s; retrying with page size 100.",
resolved_page_size,
)
users = await self._fetch_all_panel_users_pages(
page_size=100,
log_responses=log_responses,
)
return users
async def _fetch_all_panel_users_pages(
self, page_size: int, log_responses: bool = False
) -> Optional[List[Dict[str, Any]]]:
all_users = []
start_offset = 0
backend/config/settings.py
+3
View File
@@ -99,8 +99,11 @@ class Settings(BaseSettings):
PANEL_USER_CACHE_TTL_SECONDS: int = Field(default=5)
PANEL_DEVICES_CACHE_TTL_SECONDS: int = Field(default=5)
PANEL_ALL_USERS_CACHE_TTL_SECONDS: int = Field(default=5)
PANEL_ALL_USERS_PAGE_SIZE: int = Field(default=1000)
ADMIN_PANEL_STATS_CACHE_TTL_SECONDS: int = Field(default=15)
PROFILE_SYNC_CACHE_TTL_SECONDS: int = Field(default=900)
PANEL_SYNC_LIFETIME_TRAFFIC_MIN_INTERVAL_SECONDS: int = Field(default=3600)
PANEL_SYNC_LIFETIME_TRAFFIC_MIN_DELTA_BYTES: int = Field(default=104857600)
WEBAPP_RATE_LIMIT_TTL_SECONDS: int = Field(default=60)
WEBAPP_RATE_LIMIT_MAX_REQUESTS: int = Field(default=30)
WEBHOOK_QUEUE_NAME: str = Field(default="webhook-events")
backend/db/dal/user_dal.py
+4
View File
@@ -404,6 +404,10 @@ async def merge_users(
target.lifetime_used_traffic_bytes = (
target.lifetime_used_traffic_bytes or 0
) + source.lifetime_used_traffic_bytes
source_synced_at = getattr(source, "lifetime_used_traffic_synced_at", None)
target_synced_at = getattr(target, "lifetime_used_traffic_synced_at", None)
if source_synced_at and (not target_synced_at or source_synced_at > target_synced_at):
target.lifetime_used_traffic_synced_at = source_synced_at
if not target.referred_by_id and source.referred_by_id != target_user_id:
target.referred_by_id = source.referred_by_id
if target.referred_by_id == source_user_id:
backend/db/migrator.py
+14
View File
@@ -873,6 +873,15 @@ def _migration_0025_add_support_notification_timestamps(connection: Connection)
)
def _migration_0026_add_lifetime_traffic_synced_at(connection: Connection) -> None:
inspector = inspect(connection)
columns: Set[str] = {col["name"] for col in inspector.get_columns("users")}
if "lifetime_used_traffic_synced_at" not in columns:
connection.execute(
text("ALTER TABLE users ADD COLUMN lifetime_used_traffic_synced_at TIMESTAMPTZ")
)
MIGRATIONS: List[Migration] = [
Migration(
id="0001_add_channel_subscription_fields",
@@ -1010,6 +1019,11 @@ MIGRATIONS: List[Migration] = [
description="Track support ticket admin notification cooldown timestamps",
upgrade=_migration_0025_add_support_notification_timestamps,
),
Migration(
id="0026_add_lifetime_traffic_synced_at",
description="Track when lifetime traffic usage was last synced from panel",
upgrade=_migration_0026_add_lifetime_traffic_synced_at,
),
]
backend/db/models.py
+1
View File
@@ -42,6 +42,7 @@ class User(Base):
referral_code = Column(String(16), nullable=True, unique=True, index=True)
referred_by_id = Column(BigInteger, ForeignKey("users.user_id"), nullable=True)
lifetime_used_traffic_bytes = Column(BigInteger, nullable=True)
lifetime_used_traffic_synced_at = Column(DateTime(timezone=True), nullable=True)
channel_subscription_verified = Column(Boolean, nullable=True)
channel_subscription_checked_at = Column(DateTime(timezone=True), nullable=True)
channel_subscription_verified_for = Column(BigInteger, nullable=True)