fix: preserve subscriptions on panel lookup failures

2026-05-29 21:46:42 +03:00
parent 5e257c0d3a
commit 001e54cfe2
6 changed files with 564 additions and 7 deletions
@@ -328,12 +328,112 @@ class PanelApiService:
    async def _get_user_by_uuid_uncached(
        self, user_uuid: str, log_response: bool = False
    ) -> Optional[Dict[str, Any]]:
+        lookup = await self.get_user_by_uuid_lookup(user_uuid, log_response=log_response)
+        if lookup.get("ok") and isinstance(lookup.get("user"), dict):
+            return lookup["user"]
+        return None
+
+    @staticmethod
+    def _panel_response_details(response_data: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+        if not isinstance(response_data, dict):
+            return {}
+        details = response_data.get("details")
+        return details if isinstance(details, dict) else {}
+
+    @classmethod
+    def _panel_response_error_code(cls, response_data: Optional[Dict[str, Any]]) -> Optional[str]:
+        if not isinstance(response_data, dict):
+            return None
+        details = cls._panel_response_details(response_data)
+        error_code = (
+            response_data.get("errorCode")
+            or response_data.get("code")
+            or details.get("errorCode")
+            or details.get("code")
+        )
+        return str(error_code) if error_code else None
+
+    @classmethod
+    def _panel_response_message(cls, response_data: Optional[Dict[str, Any]]) -> Optional[str]:
+        if not isinstance(response_data, dict):
+            return None
+        details = cls._panel_response_details(response_data)
+        message = (
+            response_data.get("message")
+            or details.get("message")
+            or details.get("error")
+            or details.get("raw_response_text")
+        )
+        if message is None:
+            return None
+        message = str(message).replace("\n", " ").strip()
+        return message[:500] if message else None
+
+    @classmethod
+    def _is_user_not_found_response(cls, response_data: Optional[Dict[str, Any]]) -> bool:
+        if not isinstance(response_data, dict):
+            return False
+        status_code = response_data.get("status_code")
+        error_code = cls._panel_response_error_code(response_data)
+        if error_code in {"A040", "A062", "USER_NOT_FOUND", "NOT_FOUND"}:
+            return True
+        return status_code == 404
+
+    @classmethod
+    def _describe_user_lookup_failure(
+        cls,
+        response_data: Optional[Dict[str, Any]],
+        *,
+        not_found: bool,
+    ) -> str:
+        if not isinstance(response_data, dict):
+            return "classification=panel_lookup_failed response=empty"
+
+        classification = "confirmed_not_found" if not_found else "panel_lookup_failed"
+        parts = [f"classification={classification}"]
+        status_code = response_data.get("status_code")
+        if status_code is not None:
+            parts.append(f"status_code={status_code}")
+        error_code = cls._panel_response_error_code(response_data)
+        if error_code:
+            parts.append(f"error_code={error_code}")
+        message = cls._panel_response_message(response_data)
+        if message:
+            parts.append(f"message={message}")
+        return " ".join(parts)
+
+    async def get_user_by_uuid_lookup(
+        self, user_uuid: str, log_response: bool = False
+    ) -> Dict[str, Any]:
+        """Fetch a panel user and preserve whether a miss was confirmed.
+
+        ``get_user_by_uuid`` historically returned ``None`` both for a real
+        404/not-found and for transient panel/API failures. Callers that may
+        mutate local state need this richer result to avoid treating an outage
+        as a deleted panel user.
+        """
        endpoint = f"/users/{user_uuid}"
        full_response = await self._request("GET", endpoint, log_full_response=log_response)
        if full_response and not full_response.get("error") and "response" in full_response:
-            return full_response.get("response")
+            return {
+                "ok": True,
+                "user": full_response.get("response"),
+                "not_found": False,
+                "failure_reason": None,
+                "response": full_response,
+            }

-        return None
+        not_found = self._is_user_not_found_response(full_response)
+        return {
+            "ok": False,
+            "user": None,
+            "not_found": not_found,
+            "failure_reason": self._describe_user_lookup_failure(
+                full_response,
+                not_found=not_found,
+            ),
+            "response": full_response,
+        }

    async def get_user(
        self,
@@ -4,6 +4,7 @@ from datetime import datetime, timezone
 from typing import Optional

 from aiogram import Bot
+from aiogram.exceptions import TelegramBadRequest, TelegramForbiddenError
 from aiogram.types import InlineKeyboardMarkup
 from sqlalchemy.ext.asyncio import AsyncSession

@@ -125,6 +126,35 @@ class SubscriptionLifecycleNotificationService:
            return False
        try:
            await self.bot.send_message(chat_id, message_text, reply_markup=markup)
+        except (TelegramBadRequest, TelegramForbiddenError) as exc:
+            if self._is_terminal_telegram_delivery_error(exc):
+                logging.warning(
+                    "Skipping subscription notification %s for unreachable Telegram user %s: %s",
+                    stage.key,
+                    chat_id,
+                    exc,
+                )
+                try:
+                    await subscription_dal.record_subscription_notification(
+                        session,
+                        sub.subscription_id,
+                        self._channel_key(stage.key, "telegram"),
+                        sent_at=sent_at,
+                    )
+                except Exception:
+                    logging.exception(
+                        "Failed to record skipped subscription notification %s "
+                        "for Telegram user %s",
+                        stage.key,
+                        chat_id,
+                    )
+                return False
+            logging.exception(
+                "Failed to send subscription notification %s to Telegram user %s",
+                stage.key,
+                chat_id,
+            )
+            return False
        except Exception:
            logging.exception(
                "Failed to send subscription notification %s to Telegram user %s",
@@ -230,6 +260,23 @@ class SubscriptionLifecycleNotificationService:
                return chat_id
        return None

+    @staticmethod
+    def _is_terminal_telegram_delivery_error(
+        exc: TelegramBadRequest | TelegramForbiddenError,
+    ) -> bool:
+        if isinstance(exc, TelegramForbiddenError):
+            return True
+        message = str(exc).lower()
+        return any(
+            token in message
+            for token in (
+                "chat not found",
+                "bot was blocked",
+                "bot can't initiate conversation",
+                "user is deactivated",
+            )
+        )
+
    @staticmethod
    def _as_utc(value: Optional[datetime]) -> Optional[datetime]:
        if value is None:
@@ -3,6 +3,142 @@ from ._runtime import *  # noqa: F403,F405


 class SubscriptionLifecycleMixin:
+    async def _lookup_panel_user_for_subscription_details(
+        self,
+        panel_user_uuid: str,
+    ) -> Tuple[Optional[Dict[str, Any]], bool, str]:
+        lookup_method = getattr(self.panel_service, "get_user_by_uuid_lookup", None)
+        if callable(lookup_method):
+            try:
+                lookup = await lookup_method(panel_user_uuid, log_response=False)
+            except TypeError:
+                try:
+                    lookup = await lookup_method(panel_user_uuid)
+                except Exception as exc:
+                    logging.exception(
+                        "Failed to fetch panel user %s for subscription details",
+                        panel_user_uuid,
+                    )
+                    return None, False, self._panel_lookup_exception_reason(exc)
+            except Exception as exc:
+                logging.exception(
+                    "Failed to fetch panel user %s for subscription details",
+                    panel_user_uuid,
+                )
+                return None, False, self._panel_lookup_exception_reason(exc)
+
+            if isinstance(lookup, dict) and ("ok" in lookup or "not_found" in lookup):
+                user = lookup.get("user")
+                if lookup.get("ok") and isinstance(user, dict):
+                    return user, False, ""
+                reason = str(lookup.get("failure_reason") or "classification=panel_lookup_failed")
+                return None, bool(lookup.get("not_found")), reason
+
+        try:
+            panel_user = await self.panel_service.get_user_by_uuid(panel_user_uuid)
+        except Exception as exc:
+            logging.exception(
+                "Failed to fetch panel user %s for subscription details",
+                panel_user_uuid,
+            )
+            return None, False, self._panel_lookup_exception_reason(exc)
+        return (panel_user if isinstance(panel_user, dict) else None), False, ""
+
+    @staticmethod
+    def _panel_lookup_exception_reason(exc: Exception) -> str:
+        message = str(exc).replace("\n", " ").strip()
+        if len(message) > 300:
+            message = f"{message[:300]}..."
+        reason = f"classification=panel_lookup_failed exception={type(exc).__name__}"
+        if message:
+            reason = f"{reason} message={message}"
+        return reason
+
+    async def _local_active_subscription_details_fallback(
+        self,
+        db_user: User,
+        local_active_sub: Subscription,
+    ) -> Dict[str, Any]:
+        panel_sub_id = str(local_active_sub.panel_subscription_uuid or "").strip()
+        config_link_raw = (
+            await self.panel_service.get_subscription_link(panel_sub_id) if panel_sub_id else None
+        )
+        display_link, connect_button_url = await prepare_config_links(
+            self.settings,
+            config_link_raw,
+        )
+        tariff = None
+        if local_active_sub.tariff_key and self._tariffs_config():
+            try:
+                tariff = self._resolve_tariff(local_active_sub.tariff_key)
+            except Exception:
+                tariff = None
+        language = db_user.language_code or self.settings.DEFAULT_LANGUAGE
+        premium_access = (
+            await self.premium_access_for_tariff(tariff)
+            if tariff
+            else {"squad_uuids": [], "squad_labels": [], "node_labels": []}
+        )
+        premium_baseline = int(local_active_sub.premium_baseline_bytes or 0)
+        premium_topup_balance = int(local_active_sub.premium_topup_balance_bytes or 0)
+        premium_topup_used = int(getattr(local_active_sub, "premium_topup_used_bytes", 0) or 0)
+        premium_bonus_bytes = int(getattr(local_active_sub, "premium_bonus_bytes", 0) or 0)
+        return {
+            "user_id": db_user.panel_user_uuid,
+            "panel_subscription_uuid": local_active_sub.panel_subscription_uuid,
+            "panel_short_uuid": local_active_sub.panel_subscription_uuid,
+            "end_date": local_active_sub.end_date,
+            "status_from_panel": local_active_sub.status_from_panel or "LOCAL_CACHE",
+            "config_link": display_link,
+            "connect_button_url": connect_button_url,
+            "traffic_limit_bytes": local_active_sub.traffic_limit_bytes,
+            "traffic_used_bytes": local_active_sub.traffic_used_bytes,
+            "traffic_limit_strategy": "",
+            "tariff_key": local_active_sub.tariff_key,
+            "tariff_name": tariff.name(language) if tariff else None,
+            "tariff_description": tariff.description(language) if tariff else None,
+            "premium_title": tariff.premium_name(language) if tariff else None,
+            "billing_model": tariff.billing_model
+            if tariff
+            else ("traffic" if getattr(self.settings, "traffic_sale_mode", False) else "period"),
+            "tier_baseline_bytes": local_active_sub.tier_baseline_bytes,
+            "topup_balance_bytes": local_active_sub.topup_balance_bytes,
+            "regular_bonus_bytes": int(getattr(local_active_sub, "regular_bonus_bytes", 0) or 0),
+            "regular_unlimited_override": bool(
+                getattr(local_active_sub, "regular_unlimited_override", False)
+            ),
+            "premium_baseline_bytes": premium_baseline,
+            "premium_topup_balance_bytes": premium_topup_balance,
+            "premium_topup_used_bytes": premium_topup_used,
+            "premium_used_bytes": local_active_sub.premium_used_bytes,
+            "premium_bonus_bytes": premium_bonus_bytes,
+            "premium_unlimited_override": bool(
+                getattr(local_active_sub, "premium_unlimited_override", False)
+            ),
+            "premium_limit_bytes": self._premium_effective_limit_bytes(
+                premium_baseline,
+                premium_topup_balance,
+                premium_topup_used,
+                premium_bonus_bytes,
+            ),
+            "premium_is_limited": bool(local_active_sub.premium_is_limited),
+            "premium_period_start_at": getattr(local_active_sub, "premium_period_start_at", None),
+            "premium_squad_labels": premium_access.get("squad_labels") or [],
+            "premium_node_labels": premium_access.get("node_labels") or [],
+            "period_start_at": local_active_sub.period_start_at,
+            "is_throttled": bool(local_active_sub.is_throttled),
+            "base_hwid_device_limit": local_active_sub.hwid_device_limit,
+            "extra_hwid_devices": int(local_active_sub.extra_hwid_devices or 0),
+            "extra_hwid_devices_valid_until": None,
+            "extra_hwid_devices_next_valid_from": None,
+            "user_bot_username": db_user.username,
+            "is_panel_data": False,
+            "max_devices": self._effective_hwid_limit(
+                local_active_sub.hwid_device_limit,
+                int(local_active_sub.extra_hwid_devices or 0),
+            ),
+        }
+
    async def switch_tariff_without_payment(
        self,
        session: AsyncSession,
@@ -663,14 +799,34 @@ class SubscriptionLifecycleMixin:
        local_active_sub = await subscription_dal.get_active_subscription_by_user_id(
            session, user_id, panel_user_uuid
        )
-        panel_user_data = await self.panel_service.get_user_by_uuid(panel_user_uuid)
+        panel_user_data, panel_user_confirmed_absent, panel_lookup_failure_reason = (
+            await self._lookup_panel_user_for_subscription_details(panel_user_uuid)
+        )

        if not panel_user_data:
+            if panel_user_confirmed_absent:
+                logging.warning(
+                    "Panel user %s confirmed absent on panel for user %s. "
+                    "Clearing local linkage. reason=%s",
+                    panel_user_uuid,
+                    user_id,
+                    panel_lookup_failure_reason,
+                )
+                await subscription_dal.deactivate_all_user_subscriptions(session, user_id)
+                await user_dal.update_user(session, user_id, {"panel_user_uuid": None})
+                return None
            logging.warning(
-                f"Panel user {panel_user_uuid} not found on panel for user {user_id}. Clearing local linkage."  # noqa: E501
+                "Panel user %s lookup failed for user %s; treating it as a panel access/API "
+                "problem and preserving local linkage/subscription. reason=%s",
+                panel_user_uuid,
+                user_id,
+                panel_lookup_failure_reason,
            )
-            await subscription_dal.deactivate_all_user_subscriptions(session, user_id)
-            await user_dal.update_user(session, user_id, {"panel_user_uuid": None})
+            if local_active_sub:
+                return await self._local_active_subscription_details_fallback(
+                    db_user,
+                    local_active_sub,
+                )
            return None

        panel_lifetime_used = self._extract_lifetime_used_traffic(panel_user_data)