refactor: project architecture refactor, container splitting

This commit is contained in:
3252a8
2026-05-17 00:01:28 +03:00
parent e0b5218037
commit 30fb774d93
367 changed files with 2609 additions and 864 deletions
+724
View File
@@ -0,0 +1,724 @@
import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional, Union
from aiogram import Bot, Router, types
from aiogram.filters import Command
from sqlalchemy import or_, update
from sqlalchemy.ext.asyncio import AsyncSession
from bot.middlewares.i18n import JsonI18n
from bot.services.notification_service import NotificationService
from bot.services.panel_api_service import PanelApiService
from config.settings import Settings
from db.dal import panel_sync_dal, subscription_dal, user_dal
from db.models import Subscription
router = Router(name="admin_sync_router")
# Single-flight guard: panel sync runs concurrently with the bot, but only one
# sync at a time. Overlapping callers (startup, /sync, admin API) return early
# instead of queueing behind the running sync.
_sync_lock = asyncio.Lock()
def _normalize_panel_email(value: Optional[str]) -> Optional[str]:
email = (value or "").strip().lower()
return email or None
def _extract_lifetime_used_traffic_bytes(panel_user_data: dict) -> Optional[int]:
user_traffic = panel_user_data.get("userTraffic") or {}
raw_value = (
user_traffic.get("lifetimeUsedTrafficBytes") if isinstance(user_traffic, dict) else None
)
if raw_value is None:
raw_value = panel_user_data.get("lifetimeUsedTrafficBytes")
try:
if raw_value is None:
return None
return int(raw_value)
except (TypeError, ValueError):
return None
async def _bind_panel_email_to_user(
session: AsyncSession,
*,
existing_user,
email_from_panel: Optional[str],
panel_uuid: str,
) -> tuple[object, bool]:
"""Bind panel email to a local user without violating the unique email index.
Panel email is treated as verified because it comes from the operator-managed
panel. If the same email already belongs to an email-only local account for
this panel user, merge that account into the Telegram/local user.
"""
if not email_from_panel:
return existing_user, False
if existing_user.email == email_from_panel:
if not existing_user.email_verified_at:
existing_user.email_verified_at = datetime.now(timezone.utc)
return existing_user, True
return existing_user, False
user_with_email = await user_dal.get_user_by_email(session, email_from_panel)
if user_with_email and user_with_email.user_id != existing_user.user_id:
can_merge_email_identity = (
not user_with_email.telegram_id
and user_with_email.panel_user_uuid in (None, panel_uuid)
and (not existing_user.email or existing_user.email == email_from_panel)
)
if can_merge_email_identity:
try:
merged_user = await user_dal.merge_users(
session,
source_user_id=user_with_email.user_id,
target_user_id=existing_user.user_id,
)
if not merged_user.email:
merged_user.email = email_from_panel
if not merged_user.email_verified_at:
merged_user.email_verified_at = datetime.now(timezone.utc)
logging.info(
"Merged email-only user %s into user %s while binding panel email %s for panel UUID %s.", # noqa: E501
user_with_email.user_id,
merged_user.user_id,
email_from_panel,
panel_uuid,
)
return merged_user, True
except Exception as merge_error:
logging.warning(
"Could not merge email-only user %s into user %s for panel email %s: %s",
user_with_email.user_id,
existing_user.user_id,
email_from_panel,
merge_error,
)
return existing_user, False
logging.warning(
"Panel email %s for panel UUID %s is already linked to local user %s; "
"skipping email binding for user %s.",
email_from_panel,
panel_uuid,
user_with_email.user_id,
existing_user.user_id,
)
return existing_user, False
existing_user.email = email_from_panel
existing_user.email_verified_at = datetime.now(timezone.utc)
logging.info(
"Bound panel email %s to local user %s for panel UUID %s.",
email_from_panel,
existing_user.user_id,
panel_uuid,
)
return existing_user, True
async def perform_sync(
panel_service: PanelApiService,
session: AsyncSession,
settings: Settings,
i18n_instance: JsonI18n,
) -> dict:
"""Single-flight entry point — skips when another sync is already running."""
if _sync_lock.locked():
logging.info("perform_sync: skipped because another sync is already in progress")
return {
"status": "skipped",
"details": "Another sync run is already in progress.",
"errors": [],
"users_processed": 0,
"subs_synced": 0,
}
async with _sync_lock:
return await _perform_sync_impl(
panel_service=panel_service,
session=session,
settings=settings,
i18n_instance=i18n_instance,
)
async def _perform_sync_impl(
panel_service: PanelApiService,
session: AsyncSession,
settings: Settings,
i18n_instance: JsonI18n,
) -> dict:
"""
Perform panel synchronization and return results
Returns dict with status, details, and sync statistics
"""
panel_records_checked = 0
users_found_in_db = 0
users_updated = 0
subscriptions_synced_count = 0
sync_errors = []
# Additional counters for detailed logging
users_without_telegram_id = 0
users_not_found_in_db = 0
users_created = 0
users_uuid_updated = 0
subscriptions_created = 0
subscriptions_updated = 0
try:
panel_users_data = await panel_service.get_all_panel_users()
if panel_users_data is None:
error_msg = "Failed to fetch users from panel or panel API issue."
sync_errors.append(error_msg)
await panel_sync_dal.update_panel_sync_status(session, "failed", error_msg)
await session.commit()
return {"status": "failed", "details": error_msg, "errors": sync_errors}
if not panel_users_data:
status_msg = "No users found in the panel to sync."
await panel_sync_dal.update_panel_sync_status(session, "success", status_msg, 0, 0)
await session.commit()
return {
"status": "success",
"details": status_msg,
"users_synced": 0,
"subs_synced": 0,
}
total_panel_users = len(panel_users_data)
logging.info(f"Starting sync for {total_panel_users} panel users.")
for panel_user_dict in panel_users_data:
try:
panel_records_checked += 1
panel_uuid = panel_user_dict.get("uuid")
panel_user_dict.get("subscriptionUuid") or panel_user_dict.get("shortUuid")
telegram_id_from_panel = panel_user_dict.get("telegramId")
email_from_panel = _normalize_panel_email(panel_user_dict.get("email"))
if not panel_uuid:
sync_errors.append(f"Panel user missing UUID: {panel_user_dict}")
logging.warning(f"Skipping panel user without UUID: {panel_user_dict}")
continue
# Track users without telegram ID
if not telegram_id_from_panel:
users_without_telegram_id += 1
# Try to find existing user in local DB
existing_user = None
# First, try to find by telegram ID if available
if telegram_id_from_panel:
existing_user = await user_dal.get_user_by_telegram_id(
session, telegram_id_from_panel
)
if not existing_user:
existing_user = await user_dal.get_user_by_id(
session, telegram_id_from_panel
)
if existing_user:
logging.debug(f"Found user by telegramId {telegram_id_from_panel}")
# If not found by telegram ID, try to find by panel UUID.
# The panel UUID is the strongest local link for subscription sync.
if not existing_user:
existing_user = await user_dal.get_user_by_panel_uuid(session, panel_uuid)
if existing_user:
logging.debug(
f"Found user by panel UUID {panel_uuid}, telegramId: {existing_user.user_id}" # noqa: E501
)
# Update telegram ID if it was missing in panel data but we have local user
if (
telegram_id_from_panel
and existing_user.user_id != telegram_id_from_panel
):
logging.warning(
f"TelegramId mismatch: panel={telegram_id_from_panel}, local={existing_user.user_id}" # noqa: E501
)
# Finally, fall back to email. This mainly catches panel users that
# were first imported as email-only identities.
if not existing_user and email_from_panel:
existing_user = await user_dal.get_user_by_email(session, email_from_panel)
if existing_user:
logging.debug(f"Found user by email {email_from_panel}")
if not existing_user:
users_not_found_in_db += 1
if telegram_id_from_panel:
# Create new user if they have telegram_id
try:
user_data = {
"user_id": telegram_id_from_panel,
"telegram_id": telegram_id_from_panel,
"email": email_from_panel,
"email_verified_at": (
datetime.now(timezone.utc) if email_from_panel else None
),
"username": None, # Username will be updated when user interacts with bot # noqa: E501
"first_name": None, # Panel doesn't provide this info
"last_name": None, # Panel doesn't provide this info
"language_code": "ru", # Default language
"panel_user_uuid": panel_uuid,
"is_banned": False,
"referred_by_id": None,
}
new_user, was_created = await user_dal.create_user(session, user_data)
if was_created:
users_created += 1
logging.info(
f"Created new user {telegram_id_from_panel} from panel sync with UUID {panel_uuid}" # noqa: E501
)
existing_user = new_user
except Exception as e_create:
sync_errors.append(
f"Error creating user {telegram_id_from_panel}: {str(e_create)}"
)
logging.error(
f"Error creating user {telegram_id_from_panel}: {e_create}"
)
continue
elif email_from_panel:
try:
new_user, was_created = await user_dal.create_email_user(
session,
email=email_from_panel,
language_code="ru",
)
new_user.panel_user_uuid = panel_uuid
if was_created:
users_created += 1
logging.info(
f"Created new email user {new_user.user_id} from panel sync with UUID {panel_uuid}" # noqa: E501
)
existing_user = new_user
except Exception as e_create_email:
sync_errors.append(
f"Error creating email user {email_from_panel}: {str(e_create_email)}" # noqa: E501
)
logging.error(
f"Error creating email user {email_from_panel}: {e_create_email}"
)
continue
else:
logging.debug(
f"Panel user with UUID {panel_uuid} (no telegramId) not found in local DB - skipping" # noqa: E501
)
continue
# User found in local DB
users_found_in_db += 1
user_was_updated = False
# Get the actual user_id for subscription operations
actual_user_id = existing_user.user_id
# Update panel UUID if different
if existing_user.panel_user_uuid != panel_uuid:
existing_user.panel_user_uuid = panel_uuid
user_was_updated = True
users_uuid_updated += 1
logging.info(f"Updated panel UUID for user {actual_user_id}: {panel_uuid}")
existing_user, email_was_bound = await _bind_panel_email_to_user(
session,
existing_user=existing_user,
email_from_panel=email_from_panel,
panel_uuid=panel_uuid,
)
if email_was_bound:
user_was_updated = True
if telegram_id_from_panel and existing_user.telegram_id != telegram_id_from_panel:
existing_user.telegram_id = telegram_id_from_panel
user_was_updated = True
lifetime_used = _extract_lifetime_used_traffic_bytes(panel_user_dict)
if (
lifetime_used is not None
and existing_user.lifetime_used_traffic_bytes != lifetime_used
):
existing_user.lifetime_used_traffic_bytes = lifetime_used
user_was_updated = True
# Ensure panel description contains Telegram fields
try:
if panel_uuid and existing_user:
description_text = "\n".join(
line
for line in [
existing_user.email or "",
existing_user.username or "",
existing_user.first_name or "",
existing_user.last_name or "",
]
if line
)
# Update description only when it differs from the current one on panel
current_panel_description = (
panel_user_dict.get("description") or ""
).strip()
desired_description = description_text.strip()
if desired_description and desired_description != current_panel_description:
await panel_service.update_user_details_on_panel(
panel_uuid,
{
"description": description_text,
**(
{"email": existing_user.email}
if existing_user.email
else {}
),
**(
{"telegramId": existing_user.telegram_id}
if existing_user.telegram_id
else {}
),
},
)
except Exception as e_desc:
logging.warning(
f"Sync: Failed to update description for panel user {panel_uuid} (tg {actual_user_id}): {e_desc}" # noqa: E501
)
# Sync subscription data
panel_expire_at_iso = panel_user_dict.get("expireAt")
panel_status = panel_user_dict.get("status", "UNKNOWN")
if panel_expire_at_iso:
try:
panel_expire_at = datetime.fromisoformat(
panel_expire_at_iso.replace("Z", "+00:00")
)
# Prefer syncing by concrete subscription UUID (shortUuid/subscriptionUuid)
subscription_uuid_from_panel = panel_user_dict.get(
"subscriptionUuid"
) or panel_user_dict.get("shortUuid")
if subscription_uuid_from_panel:
# Если панель говорит, что подписка ACTIVE — сначала деактивируем все другие активные # noqa: E501
if panel_status == "ACTIVE":
await session.execute(
update(Subscription)
.where(
Subscription.panel_user_uuid == panel_uuid,
Subscription.is_active.is_(True),
or_(
Subscription.panel_subscription_uuid
!= subscription_uuid_from_panel,
Subscription.panel_subscription_uuid.is_(None),
),
)
.values(
is_active=False,
status_from_panel="INACTIVE",
)
)
# Try to find subscription by its panel_subscription_uuid first (idempotent) # noqa: E501
existing_sub_by_uuid = (
await subscription_dal.get_subscription_by_panel_subscription_uuid(
session, subscription_uuid_from_panel
)
)
if existing_sub_by_uuid:
# Atomic update of all relevant fields
await subscription_dal.update_subscription(
session,
existing_sub_by_uuid.subscription_id,
{
"user_id": actual_user_id,
"panel_user_uuid": panel_uuid,
"end_date": panel_expire_at,
"is_active": panel_status == "ACTIVE",
"status_from_panel": panel_status,
},
)
subscriptions_synced_count += 1
subscriptions_updated += 1
user_was_updated = True
logging.debug(
f"Synced existing subscription {existing_sub_by_uuid.subscription_id} " # noqa: E501
f"for user {actual_user_id}: expires {panel_expire_at}, status {panel_status}" # noqa: E501
)
else:
# Create a new subscription only when we have a concrete subscription UUID # noqa: E501
sub_payload = {
"user_id": actual_user_id,
"panel_user_uuid": panel_uuid,
"panel_subscription_uuid": subscription_uuid_from_panel,
# Do not guess precise start_date from panel; keep nullable
"start_date": None,
"end_date": panel_expire_at,
"duration_months": None,
"is_active": panel_status == "ACTIVE",
"status_from_panel": panel_status,
"traffic_limit_bytes": settings.user_traffic_limit_bytes,
"auto_renew_enabled": False,
}
created_sub = await subscription_dal.upsert_subscription(
session, sub_payload
)
subscriptions_synced_count += 1
subscriptions_created += 1
user_was_updated = True
logging.debug(
f"Created subscription {created_sub.subscription_id} "
f"for user {actual_user_id} by panel_sub_uuid {subscription_uuid_from_panel}" # noqa: E501
)
else:
# No subscription UUID from panel: only update an already active subscription for this user/panel UUID # noqa: E501
active_sub = await subscription_dal.get_active_subscription_by_user_id(
session, actual_user_id, panel_uuid
)
if active_sub:
await subscription_dal.update_subscription(
session,
active_sub.subscription_id,
{
"end_date": panel_expire_at,
"is_active": panel_status == "ACTIVE",
"status_from_panel": panel_status,
},
)
subscriptions_synced_count += 1
subscriptions_updated += 1
user_was_updated = True
logging.debug(
f"Updated active subscription {active_sub.subscription_id} "
f"for user {actual_user_id}: expires {panel_expire_at}, status {panel_status}" # noqa: E501
)
else:
# Without a concrete subscription UUID we avoid creating new records to keep sync idempotent # noqa: E501
logging.debug(
f"No subscriptionUuid for panel user {panel_uuid}; skipped creation for user {actual_user_id}" # noqa: E501
)
except Exception as e:
sync_errors.append(
f"Error syncing subscription for user {actual_user_id}: {str(e)}"
)
logging.error(f"Error syncing subscription for user {actual_user_id}: {e}")
if user_was_updated:
users_updated += 1
except Exception as e_user:
sync_errors.append(
f"Error processing panel user {panel_user_dict.get('uuid', 'unknown')}: {str(e_user)}" # noqa: E501
)
logging.error(f"Error syncing user: {e_user}")
# Update sync status
status = "completed_with_errors" if sync_errors else "completed"
# Build additional stats
default_lang = settings.DEFAULT_LANGUAGE
additional_stats = ""
if users_without_telegram_id > 0:
additional_stats += i18n_instance.gettext(
default_lang,
"admin_sync_no_telegram_id",
count=users_without_telegram_id,
)
if users_not_found_in_db > 0:
additional_stats += i18n_instance.gettext(
default_lang,
"admin_sync_not_found_in_db",
count=users_not_found_in_db,
)
if sync_errors:
additional_stats += i18n_instance.gettext(
default_lang, "admin_sync_errors", count=len(sync_errors)
)
# Build full details using localization
details = i18n_instance.gettext(
default_lang,
"admin_sync_details",
panel_records_checked=panel_records_checked,
users_found_in_db=users_found_in_db,
users_created=users_created,
users_updated=users_updated,
subscriptions_synced_count=subscriptions_synced_count,
subscriptions_created=subscriptions_created,
subscriptions_updated=subscriptions_updated,
additional_stats=additional_stats,
)
await panel_sync_dal.update_panel_sync_status(
session,
status,
details,
panel_records_checked,
subscriptions_synced_count,
)
await session.commit()
# Detailed logging summary
logging.info("Sync completed - Summary:")
logging.info(f" Panel records checked: {panel_records_checked}")
logging.info(f" Users without telegramId: {users_without_telegram_id}")
logging.info(f" Users not found in local DB: {users_not_found_in_db}")
logging.info(f" Users found in local DB: {users_found_in_db}")
logging.info(f" Users created: {users_created}")
logging.info(f" Users with UUID updated: {users_uuid_updated}")
logging.info(f" Users updated overall: {users_updated}")
logging.info(f" Subscriptions total synced: {subscriptions_synced_count}")
logging.info(f" Subscriptions created: {subscriptions_created}")
logging.info(f" Subscriptions updated: {subscriptions_updated}")
logging.info(f" Sync errors: {len(sync_errors)}")
return {
"status": status,
"details": details,
"users_processed": panel_records_checked,
"users_synced": users_found_in_db,
"users_created": users_created,
"subs_synced": subscriptions_synced_count,
"errors": sync_errors,
}
except Exception as e_sync_global:
await session.rollback()
logging.error(f"Global error during sync: {e_sync_global}", exc_info=True)
error_detail = f"Unexpected error during sync: {str(e_sync_global)}"
await panel_sync_dal.update_panel_sync_status(
session,
"failed",
error_detail,
panel_records_checked,
subscriptions_synced_count,
)
return {
"status": "failed",
"details": error_detail,
"errors": [str(e_sync_global)],
}
@router.message(Command("sync"))
async def sync_command_handler(
message_event: Union[types.Message, types.CallbackQuery],
bot: Bot,
settings: Settings,
i18n_data: dict,
panel_service: PanelApiService,
session: AsyncSession,
):
current_lang = i18n_data.get("current_language", settings.DEFAULT_LANGUAGE)
i18n: Optional[JsonI18n] = i18n_data.get("i18n_instance")
if not i18n:
logging.error("i18n missing in sync_command_handler")
if isinstance(message_event, types.Message):
await message_event.answer("Language error.")
elif isinstance(message_event, types.CallbackQuery):
await message_event.answer("Language error.", show_alert=True)
return
_ = lambda key, **kwargs: i18n.gettext(current_lang, key, **kwargs)
target_chat_id = (
message_event.chat.id
if isinstance(message_event, types.Message)
else (message_event.message.chat.id if message_event.message else None)
)
if not target_chat_id:
logging.error("Sync handler: could not determine target_chat_id.")
if isinstance(message_event, types.CallbackQuery):
await message_event.answer("Error initiating sync.", show_alert=True)
return
if isinstance(message_event, types.Message):
await message_event.answer(_("sync_started_simple"))
logging.info(f"Admin ({message_event.from_user.id}) triggered panel sync.")
# Use the extracted perform_sync function
try:
sync_result = await perform_sync(panel_service, session, settings, i18n)
status = sync_result.get("status")
details = sync_result.get("details", "No details available")
errors = sync_result.get("errors", [])
# Simple confirmation message to admin
if status == "failed":
await bot.send_message(target_chat_id, _("sync_failed_simple"))
elif status == "completed_with_errors":
await bot.send_message(
target_chat_id,
_("sync_errors_simple", errors_count=len(errors)),
)
else:
await bot.send_message(target_chat_id, _("sync_success_simple"))
# Send notification to log channel with proper thread handling
try:
notification_service = NotificationService(bot, settings, i18n)
await notification_service.notify_panel_sync(
status,
details,
sync_result.get("users_processed", 0),
sync_result.get("subs_synced", 0),
)
except Exception as e_notification:
logging.error(f"Failed to send sync notification: {e_notification}")
except Exception as e_sync_global:
logging.error(f"Global error during /sync command: {e_sync_global}", exc_info=True)
await bot.send_message(target_chat_id, _("sync_critical_error"))
# Send notification to log channel about failure
try:
notification_service = NotificationService(bot, settings, i18n)
await notification_service.notify_panel_sync("failed", str(e_sync_global), 0, 0)
except Exception as e_notification:
logging.error(f"Failed to send sync failure notification: {e_notification}")
@router.message(Command("syncstatus"))
async def sync_status_command_handler(
message: types.Message, i18n_data: dict, settings: Settings, session: AsyncSession
):
current_lang = i18n_data.get("current_language", settings.DEFAULT_LANGUAGE)
i18n: Optional[JsonI18n] = i18n_data.get("i18n_instance")
if not i18n:
await message.answer("Language error.")
return
_ = lambda key, **kwargs: i18n.gettext(current_lang, key, **kwargs)
status_record_model = await panel_sync_dal.get_panel_sync_status(session)
response_text = ""
if status_record_model:
last_time_val = status_record_model.last_sync_time
last_time_str = last_time_val.strftime("%Y-%m-%d %H:%M:%S UTC") if last_time_val else "N/A"
details_val = status_record_model.details
details_str = details_val or "N/A"
response_text = (
f"<b>{_('admin_stats_last_sync_header')}</b>\n"
f" {_('admin_stats_sync_time')}: {last_time_str}\n"
f" {_('admin_stats_sync_status')}: {status_record_model.status}\n"
f" {_('admin_stats_sync_users_processed')}: {status_record_model.users_processed_from_panel}\n" # noqa: E501
f" {_('admin_stats_sync_subs_synced')}: {status_record_model.subscriptions_synced}\n"
f" {_('admin_stats_sync_details_label')}: {details_str}"
)
else:
response_text = _("admin_sync_status_never_run")
await message.answer(response_text, parse_mode="HTML")